From 07e8ad13f28d07c770adc2e289ce72d0efcb6a55 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 29 Sep 2026 20:37:10 +0700 Subject: [PATCH] fix: read Japanese, Chinese and Korean text in data files, imports and SQL files --- CHANGELOG.md | 15 + .../DelimitedDialectDetector.swift | 55 --- .../DelimitedFileReader.swift | 96 +++++ .../TableProTabularIO/DelimitedSource.swift | 12 +- .../TableProTabularIO/DelimitedWriter.swift | 98 ++++- .../TabularCodeUnitLayout.swift | 145 ++++++++ .../TabularEncodingDetector.swift | 265 +++++++++++++ .../TableProTabularIO/TabularJISRoman.swift | 19 + .../TableProTabularIO/TabularTextCodec.swift | 4 +- .../TabularTextEncoding.swift | 37 +- .../TabularTextTranscoder.swift | 350 +++++++++++++++++- .../TabularTranscodingMap.swift | 53 +++ .../TabularEncodingDetectorTests.swift | 212 +++++++++++ .../TabularTextTranscoderTests.swift | 149 ++++++++ .../TabularSaveTests.swift | 147 +++++++- .../CSVImportPlugin/CSVImportOptions.swift | 19 +- .../CSVImportOptionsView.swift | 7 +- .../CSVImportPlugin/CSVImportParsing.swift | 9 +- Plugins/CSVImportPlugin/CSVImportPlugin.swift | 29 +- Plugins/CSVImportPlugin/CSVImportText.swift | 87 +++++ .../JSONImportPlugin/JSONImportParsing.swift | 15 +- .../DataFiles/DataFileController+Saving.swift | 75 +++- .../Core/DataFiles/DataFileController.swift | 19 +- .../Core/DataFiles/DataFileCountPhrase.swift | 6 + .../Core/DataFiles/DataFileDocument.swift | 6 +- TablePro/Core/DataFiles/DataFileLoader.swift | 90 ++--- .../Core/DataFiles/DataFileWorkingCopy.swift | 4 - .../Core/Plugins/SqlFileImportSource.swift | 91 ++++- .../Core/Services/Export/ImportService.swift | 1 + .../Utilities/File/FileTextEncoding.swift | 6 +- .../Core/Utilities/File/FileTextLoader.swift | 61 ++- .../SQL/SQLClientEncodingRewriter.swift | 199 ++++++++++ .../Utilities/SQL/SQLDumpEncodingChoice.swift | 77 ++++ .../Core/Utilities/SQL/SQLTokenCursor.swift | 6 + .../SQL/SQLTranscodedLiteralRewriter.swift | 267 +++++++++++++ .../LinkedFileVersionHistoryProvider.swift | 2 +- TablePro/Models/Export/ImportModels.swift | 41 +- TablePro/Models/Query/FileTabBaseline.swift | 25 +- TablePro/Models/Query/QueryTab.swift | 4 +- TablePro/Models/Query/QueryTabState.swift | 6 +- TablePro/Resources/Localizable.xcstrings | 32 +- TablePro/Views/DataFiles/DataFileSheets.swift | 2 +- .../Views/DataFiles/DataFileStatusBar.swift | 12 +- TablePro/Views/Import/ImportDialog.swift | 16 +- .../Intents/IntentDataError.swift | 6 +- .../TableProMobile/Intents/RowPayload.swift | 26 +- .../TableProMobile/Localizable.xcstrings | 10 +- .../Intents/RowPayloadTests.swift | 23 ++ TableProMobile/project.yml | 1 + .../DataFiles/DataFileControllerTests.swift | 153 +++++++- .../SqlFileImportSourceEncodingTests.swift | 88 +++++ .../Core/Utilities/FileTextLoaderTests.swift | 64 ++++ .../Core/Utilities/FileTextWriterTests.swift | 9 + .../SQL/SQLClientEncodingRewriterTests.swift | 80 ++++ .../SQL/SQLDumpEncodingChoiceTests.swift | 69 ++++ .../SQLTranscodedLiteralRewriterTests.swift | 105 ++++++ .../Models/FileTabBaselineTests.swift | 79 ++++ .../CSVImportEncodingFailureTests.swift | 106 ++++++ .../Plugins/CSVImportPluginTests.swift | 129 ++++++- .../Plugins/JSONImportPluginTests.swift | 29 ++ TableProUITests/DataFileWindowUITests.swift | 27 ++ docs/external-api/ios-shortcuts.mdx | 5 +- docs/features/data-files.mdx | 8 +- docs/features/import-export.mdx | 8 +- docs/features/sql-files.mdx | 4 +- project.yml | 6 + scripts/benchmark/data-files/main.swift | 7 +- 67 files changed, 3628 insertions(+), 285 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProTabularIO/DelimitedFileReader.swift create mode 100644 Packages/TableProCore/Sources/TableProTabularIO/TabularCodeUnitLayout.swift create mode 100644 Packages/TableProCore/Sources/TableProTabularIO/TabularEncodingDetector.swift create mode 100644 Packages/TableProCore/Sources/TableProTabularIO/TabularJISRoman.swift create mode 100644 Packages/TableProCore/Sources/TableProTabularIO/TabularTranscodingMap.swift create mode 100644 Packages/TableProCore/Tests/TableProTabularIOTests/TabularEncodingDetectorTests.swift create mode 100644 Packages/TableProCore/Tests/TableProTabularIOTests/TabularTextTranscoderTests.swift create mode 100644 Plugins/CSVImportPlugin/CSVImportText.swift create mode 100644 TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift create mode 100644 TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift create mode 100644 TablePro/Core/Utilities/SQL/SQLTranscodedLiteralRewriter.swift create mode 100644 TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift create mode 100644 TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift create mode 100644 TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift create mode 100644 TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift create mode 100644 TableProTests/Plugins/CSVImportEncodingFailureTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 563e5587f7..cd69c3e7f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - SAP HANA database driver plugin. (#1966) +- Shift JIS, EUC-JP, GB 18030, Big5, EUC-KR and UTF-16 options for CSV and SQL import. ### Fixed @@ -18,6 +19,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - External Clients level and Cloudflare, Cloud SQL, SOCKS and Tunnel Command settings reset by an iCloud sync. - iOS row editor saving the placeholder of a long text or binary value over the full value. - Explain Analyze running write statements on Read-Only connections and skipping the Alert and Safe Mode confirmation. +- Japanese, Chinese and Korean text in CSV, TSV and SQL files opening as garbled characters. +- Every row of a UTF-16 data file rewritten on save. +- Latin-1 and Windows-1252 SQL dumps importing double-encoded text through their own `SET NAMES`. +- Data file window stuck on an error after reloading it with the wrong encoding. +- Encoding chosen in File Properties lost when another app changes the file. +- No columns found for a JSON Lines file over 256 KB with non-ASCII text. +- VoiceOver reading the data file status bar's delimiter and encoding as only "File format". +- CSV import writing bytes it could not read into the table as Latin-1 text. +- Data file window picking up another app's in-place writes on HFS+ and exFAT volumes. +- Wrong row number and blank-looking characters in the data file save error for text the encoding cannot store. +- Yen sign failing to save in Shift JIS and EUC-JP data files and SQL files. +- CSV import with single quotes merging rows at a double quote inside a field. +- Binary values in Latin-1 and Windows-1252 SQL dumps imported as different bytes. +- Shortcuts rejecting CSV and JSON files that are not UTF-8. ## [0.76.1] - 2026-09-29 diff --git a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedDialectDetector.swift b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedDialectDetector.swift index 9f7c9f6fbd..9146de5cbd 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedDialectDetector.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedDialectDetector.swift @@ -1,34 +1,12 @@ import Foundation public enum DelimitedDialectDetector { - public struct EncodingSniff: Equatable, Sendable { - public let encoding: TabularTextEncoding - public let byteOrderMarkLength: Int - - public var hasByteOrderMark: Bool { byteOrderMarkLength > 0 } - } - public static let layoutScanLimit = 65_536 - public static let encodingScanLimit = 262_144 private static let candidateDelimiters: [UInt8] = [ DelimitedDialect.comma, DelimitedDialect.tab, DelimitedDialect.semicolon, DelimitedDialect.pipe ] - public static func sniffEncoding(_ prefix: UnsafeBufferPointer) -> EncodingSniff { - if hasPrefix(prefix, [0xEF, 0xBB, 0xBF]) { - return EncodingSniff(encoding: .utf8, byteOrderMarkLength: 3) - } - if hasPrefix(prefix, [0xFF, 0xFE]) { - return EncodingSniff(encoding: .utf16LittleEndian, byteOrderMarkLength: 2) - } - if hasPrefix(prefix, [0xFE, 0xFF]) { - return EncodingSniff(encoding: .utf16BigEndian, byteOrderMarkLength: 2) - } - let probe = UnsafeBufferPointer(rebasing: prefix[0.., contentStart: Int, @@ -128,37 +106,4 @@ public enum DelimitedDialectDetector { } return .lf } - - private static func hasPrefix(_ bytes: UnsafeBufferPointer, _ prefix: [UInt8]) -> Bool { - guard bytes.count >= prefix.count else { return false } - for (offset, byte) in prefix.enumerated() where bytes[offset] != byte { - return false - } - return true - } - - private static func isValidUTF8Prefix(_ bytes: UnsafeBufferPointer) -> Bool { - var end = bytes.count - var continuation = 0 - while end > 0, continuation < 3, (bytes[end - 1] & 0xC0) == 0x80 { - end -= 1 - continuation += 1 - } - if end > 0, bytes[end - 1] >= 0xC0 { - end -= 1 - } - let trimmed = UnsafeBufferPointer(rebasing: bytes[0.. Void)? = nil, + isCancelled: @escaping @Sendable () -> Bool = { false } + ) async throws -> DelimitedFileContents { + let sniff = TabularEncodingDetector.sniff(original) + let encoding = dialectOverride?.encoding ?? sniff.encoding + let byteOrderMarkLength = encoding == sniff.encoding ? sniff.byteOrderMarkLength : 0 + let text = try readText( + original, + encoding: encoding, + byteOrderMarkLength: byteOrderMarkLength, + transcodedFileURL: transcodedFileURL, + progress: { progress?($0 * transcodingShare) }, + isCancelled: isCancelled + ) + if isCancelled() { throw TabularCancellation() } + let detected = text.bytes.withUnsafeBytes { raw in + DelimitedDialectDetector.detect( + raw.bindMemory(to: UInt8.self), + contentStart: text.contentStart, + encoding: encoding, + hasByteOrderMark: byteOrderMarkLength > 0, + fileExtension: fileExtension + ) + } + let buildStart = text.origin == nil ? 0 : transcodingShare + let source = try await DelimitedSourceBuilder.build( + bytes: text.bytes, + dialect: dialectOverride ?? detected, + byteEncoding: text.byteEncoding, + contentStart: text.contentStart, + origin: text.origin, + progress: { progress?(buildStart + $0 * (0.95 - buildStart)) }, + isCancelled: isCancelled + ) + return DelimitedFileContents(source: source, undecodableLineCount: text.undecodableLineCount) + } + + private struct Text { + let bytes: Data + let byteEncoding: TabularTextEncoding + let contentStart: Int + let origin: TabularTranscodingMap? + let undecodableLineCount: Int + } + + private static func readText( + _ original: Data, + encoding: TabularTextEncoding, + byteOrderMarkLength: Int, + transcodedFileURL: URL, + progress: (Double) -> Void, + isCancelled: () -> Bool + ) throws -> Text { + guard !encoding.readsInPlace else { + return Text( + bytes: original, + byteEncoding: encoding, + contentStart: byteOrderMarkLength, + origin: nil, + undecodableLineCount: 0 + ) + } + let transcoded = try TabularTextTranscoder.transcode( + original, + from: encoding, + skippingPrefix: byteOrderMarkLength, + to: transcodedFileURL, + progress: progress, + isCancelled: isCancelled + ) + return Text( + bytes: try Data(contentsOf: transcodedFileURL, options: .alwaysMapped), + byteEncoding: .utf8, + contentStart: 0, + origin: transcoded.map, + undecodableLineCount: transcoded.undecodableLineCount + ) + } +} diff --git a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedSource.swift b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedSource.swift index 51266a1af8..d98a6a9243 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedSource.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedSource.swift @@ -8,6 +8,7 @@ public struct DelimitedSource: TabularSource { public let fieldCountOfFirstRow: Int public let maximumFieldCount: Int public let raggedRowCount: Int + public let origin: TabularTranscodingMap? public init( bytes: Data, @@ -16,7 +17,8 @@ public struct DelimitedSource: TabularSource { byteEncoding: TabularTextEncoding, fieldCountOfFirstRow: Int, maximumFieldCount: Int, - raggedRowCount: Int + raggedRowCount: Int, + origin: TabularTranscodingMap? = nil ) { self.bytes = bytes self.index = index @@ -25,6 +27,7 @@ public struct DelimitedSource: TabularSource { self.fieldCountOfFirstRow = fieldCountOfFirstRow self.maximumFieldCount = maximumFieldCount self.raggedRowCount = raggedRowCount + self.origin = origin } public var rowCount: Int { index.rowCount } @@ -47,7 +50,8 @@ public struct DelimitedSource: TabularSource { byteEncoding: byteEncoding, fieldCountOfFirstRow: fieldCountOfFirstRow, maximumFieldCount: maximumFieldCount, - raggedRowCount: raggedRowCount + raggedRowCount: raggedRowCount, + origin: origin ) } @@ -141,6 +145,7 @@ public enum DelimitedSourceBuilder { dialect: DelimitedDialect, byteEncoding: TabularTextEncoding, contentStart: Int, + origin: TabularTranscodingMap? = nil, progress: (@Sendable (Double) -> Void)? = nil, isCancelled: @escaping @Sendable () -> Bool = { false } ) async throws -> DelimitedSource { @@ -168,7 +173,8 @@ public enum DelimitedSourceBuilder { byteEncoding: byteEncoding, fieldCountOfFirstRow: counts.first, maximumFieldCount: counts.maximum, - raggedRowCount: counts.ragged + raggedRowCount: counts.ragged, + origin: origin ) } diff --git a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedWriter.swift b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedWriter.swift index a5e0d08738..bb8407bf00 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/DelimitedWriter.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/DelimitedWriter.swift @@ -22,13 +22,36 @@ public struct DelimitedWriter { self.source = source } + private enum SourceRowCopy { + case sourceBytes + case originalBytes(TabularTranscodingMap) + case fields + } + public var canCopySourceBytes: Bool { - guard let source else { return false } - return source.byteEncoding == dialect.encoding - && source.dialect.delimiter == dialect.delimiter - && source.dialect.quote == dialect.quote - && source.dialect.escape == dialect.escape - && source.dialect.lineEnding == dialect.lineEnding + switch sourceRowCopy { + case .sourceBytes, .originalBytes: + return true + case .fields: + return false + } + } + + private var sourceRowCopy: SourceRowCopy { + guard let source, + source.dialect.delimiter == dialect.delimiter, + source.dialect.quote == dialect.quote, + source.dialect.escape == dialect.escape, + source.dialect.lineEnding == dialect.lineEnding else { + return .fields + } + if source.byteEncoding == dialect.encoding { + return .sourceBytes + } + if let origin = source.origin, origin.encoding == dialect.encoding { + return .originalBytes(origin) + } + return .fields } public func write( @@ -53,13 +76,21 @@ public struct DelimitedWriter { buffer.append(contentsOf: dialect.encoding.byteOrderMark) } let lineEnding = try encodedLineEnding() + let copy = sourceRowCopy + let originalRowStarts = originalRowBoundaries(for: copy) var pendingTerminator = false var outputRow = 0 for row in rows { if pendingTerminator { buffer.append(contentsOf: lineEnding) } - pendingTerminator = try append(row, outputRow: outputRow, lineEnding: lineEnding, into: &buffer) + pendingTerminator = try append( + row, + outputRow: outputRow, + copy: copy, + originalRowStarts: originalRowStarts, + into: &buffer + ) outputRow += 1 if buffer.count >= Self.flushThreshold { if isCancelled() { throw TabularCancellation() } @@ -104,7 +135,8 @@ public struct DelimitedWriter { private func append( _ row: DelimitedOutputRow, outputRow: Int, - lineEnding: [UInt8], + copy: SourceRowCopy, + originalRowStarts: [Int], into buffer: inout [UInt8] ) throws -> Bool { switch row { @@ -113,7 +145,14 @@ public struct DelimitedWriter { return true case .source(let sourceRow): guard let source else { return false } - return try appendSourceRow(sourceRow, of: source, outputRow: outputRow, lineEnding: lineEnding, into: &buffer) + return try appendSourceRow( + sourceRow, + of: source, + outputRow: outputRow, + copy: copy, + originalRowStarts: originalRowStarts, + into: &buffer + ) } } @@ -121,20 +160,43 @@ public struct DelimitedWriter { _ sourceRow: Int, of source: DelimitedSource, outputRow: Int, - lineEnding: [UInt8], + copy: SourceRowCopy, + originalRowStarts: [Int], into buffer: inout [UInt8] ) throws -> Bool { - guard canCopySourceBytes else { + let range = source.rawRange(ofRow: sourceRow) + switch copy { + case .fields: buffer.append(contentsOf: try encodeRow(source.decodedFields(row: sourceRow), outputRow: outputRow)) return true + case .sourceBytes: + let terminatorLength = source.bytes.withUnsafeBytes { raw -> Int in + let bytes = raw.bindMemory(to: UInt8.self) + buffer.append(contentsOf: UnsafeBufferPointer(rebasing: bytes[range])) + return Self.terminatorLength(in: bytes, range: range) + } + return terminatorLength == 0 + case .originalBytes(let origin): + let start = originalRowStarts[sourceRow] + let end = max(start, originalRowStarts[sourceRow + 1]) + guard (end - start).isMultiple(of: origin.encoding.codeUnitLayout.width) else { + buffer.append(contentsOf: try encodeRow(source.decodedFields(row: sourceRow), outputRow: outputRow)) + return true + } + origin.original.withUnsafeBytes { raw in + buffer.append(contentsOf: UnsafeBufferPointer(rebasing: raw.bindMemory(to: UInt8.self)[start.. Int in - let bytes = raw.bindMemory(to: UInt8.self) - buffer.append(contentsOf: UnsafeBufferPointer(rebasing: bytes[range])) - return Self.terminatorLength(in: bytes, range: range) - } - return terminatorLength == 0 + } + + private func originalRowBoundaries(for copy: SourceRowCopy) -> [Int] { + guard case .originalBytes(let origin) = copy, let source else { return [] } + let index = source.index + return origin.originalOffsets(ofLineStarts: index.rowStarts + [index.endOffset], in: source.bytes) } private static func terminatorLength(in bytes: UnsafeBufferPointer, range: Range) -> Int { diff --git a/Packages/TableProCore/Sources/TableProTabularIO/TabularCodeUnitLayout.swift b/Packages/TableProCore/Sources/TableProTabularIO/TabularCodeUnitLayout.swift new file mode 100644 index 0000000000..9d761bd607 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProTabularIO/TabularCodeUnitLayout.swift @@ -0,0 +1,145 @@ +import Foundation + +enum TabularCodeUnitLayout: Sendable { + case asciiCompatible + case utf16LittleEndian + case utf16BigEndian + + var width: Int { + switch self { + case .asciiCompatible: return 1 + case .utf16LittleEndian, .utf16BigEndian: return 2 + } + } + + func lineBreakByte(in bytes: UnsafeBufferPointer, at offset: Int) -> UInt8? { + switch self { + case .asciiCompatible: + return Self.lineBreak(bytes[offset]) + case .utf16LittleEndian: + guard offset + 1 < bytes.count, bytes[offset + 1] == 0 else { return nil } + return Self.lineBreak(bytes[offset]) + case .utf16BigEndian: + guard offset + 1 < bytes.count, bytes[offset] == 0 else { return nil } + return Self.lineBreak(bytes[offset + 1]) + } + } + + func endOfLastLineBreak(in bytes: UnsafeBufferPointer, from start: Int, before end: Int) -> Int? { + guard end - start >= width else { return nil } + var offset = aligned(end - width, from: start) + while offset >= start { + if let lineBreak = lineBreakByte(in: bytes, at: offset) { + return endOfLineBreak(lineBreak, at: offset, in: bytes) + } + offset -= width + } + return nil + } + + func endOfLastCharacter(in bytes: UnsafeBufferPointer, from start: Int, before end: Int) -> Int? { + guard end - start >= width else { return nil } + var offset = aligned(end - width, from: start) + while offset >= start { + if endsCharacter(in: bytes, at: offset) { return offset + width } + offset -= width + } + return nil + } + + func endOfNextCharacter(in bytes: UnsafeBufferPointer, from start: Int, notBefore floor: Int) -> Int? { + var offset = max(start, aligned(floor, from: start)) + while offset + width <= bytes.count { + if endsCharacter(in: bytes, at: offset) { return offset + width } + offset += width + } + return nil + } + + func lineCount(in bytes: UnsafeBufferPointer, from start: Int, to end: Int) -> Int { + var count = 0 + var offset = start + while offset + width <= end { + if let lineBreak = lineBreakByte(in: bytes, at: offset), + lineBreak == Self.lineFeed || !isLineFeed(in: bytes, at: offset + width, before: end) { + count += 1 + } + offset += width + } + return count + } + + func endOfNextLineBreak(in bytes: UnsafeBufferPointer, from start: Int, notBefore floor: Int) -> Int? { + var offset = max(start, aligned(floor, from: start)) + while offset + width <= bytes.count { + if let lineBreak = lineBreakByte(in: bytes, at: offset) { + return endOfLineBreak(lineBreak, at: offset, in: bytes) + } + offset += width + } + return nil + } + + func lineBreakCount(in bytes: UnsafeBufferPointer, from start: Int, to end: Int) -> Int { + var count = 0 + var offset = start + while offset + width <= end { + if lineBreakByte(in: bytes, at: offset) != nil { count += 1 } + offset += width + } + return count + } + + func offset(afterLineBreaks count: Int, in bytes: UnsafeBufferPointer, from start: Int) -> Int { + guard count > 0 else { return start } + var remaining = count + var offset = start + while offset + width <= bytes.count { + if lineBreakByte(in: bytes, at: offset) != nil { + remaining -= 1 + if remaining == 0 { return offset + width } + } + offset += width + } + return bytes.count + } + + private func endOfLineBreak(_ lineBreak: UInt8, at offset: Int, in bytes: UnsafeBufferPointer) -> Int { + guard lineBreak == Self.carriageReturn, isLineFeed(in: bytes, at: offset + width, before: bytes.count) else { + return offset + width + } + return offset + 2 * width + } + + private func isLineFeed(in bytes: UnsafeBufferPointer, at offset: Int, before end: Int) -> Bool { + offset + width <= end && lineBreakByte(in: bytes, at: offset) == Self.lineFeed + } + + private func endsCharacter(in bytes: UnsafeBufferPointer, at offset: Int) -> Bool { + switch self { + case .asciiCompatible: + return bytes[offset] < Self.lowestByteInsideACharacter + case .utf16LittleEndian: + return offset + 1 < bytes.count && !Self.isHighSurrogateByte(bytes[offset + 1]) + case .utf16BigEndian: + return offset + 1 < bytes.count && !Self.isHighSurrogateByte(bytes[offset]) + } + } + + private static func isHighSurrogateByte(_ byte: UInt8) -> Bool { + (0xD8...0xDB).contains(byte) + } + + private func aligned(_ offset: Int, from start: Int) -> Int { + start + ((offset - start) / width) * width + } + + static let lowestByteInsideACharacter: UInt8 = 0x30 + + private static let lineFeed: UInt8 = 0x0A + private static let carriageReturn: UInt8 = 0x0D + + private static func lineBreak(_ byte: UInt8) -> UInt8? { + byte == lineFeed || byte == carriageReturn ? byte : nil + } +} diff --git a/Packages/TableProCore/Sources/TableProTabularIO/TabularEncodingDetector.swift b/Packages/TableProCore/Sources/TableProTabularIO/TabularEncodingDetector.swift new file mode 100644 index 0000000000..5922978825 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProTabularIO/TabularEncodingDetector.swift @@ -0,0 +1,265 @@ +import Foundation + +public struct TabularEncodingSniff: Equatable, Sendable { + public let encoding: TabularTextEncoding + public let byteOrderMarkLength: Int + + public init(encoding: TabularTextEncoding, byteOrderMarkLength: Int) { + self.encoding = encoding + self.byteOrderMarkLength = byteOrderMarkLength + } + + public var hasByteOrderMark: Bool { byteOrderMarkLength > 0 } +} + +public enum TabularEncodingDetector { + public static let sampleLength = 65_536 + + static let legacyCandidates: [TabularTextEncoding] = [.shiftJIS, .eucJP, .gb18030, .big5, .eucKR, .windows1252] + + private static let markedEncodings: [TabularTextEncoding] = [.utf8, .utf16LittleEndian, .utf16BigEndian] + private static let asciiScanLimit = 64 << 20 + private static let structuralScalars: Set = [0x09, 0x0A, 0x0D, 0x22, 0x2C, 0x3B, 0x7C] + private static let halfWidthKatakana: ClosedRange = 0xA1...0xDF + private static let halfWidthKatakanaShare = 0.9 + private static let halfWidthKatakanaRunLength = 3 + private static let halfWidthKatakanaRunShare = 0.5 + private static let undecodableLineShareLimit = 10 + private static let eastAsianScalars: [ClosedRange] = [ + 0x1100...0x11FF, 0x2460...0x24FF, 0x3000...0x303F, 0x3040...0x30FF, 0x3100...0x312F, 0x3130...0x318F, + 0x3200...0x33FF, 0x3400...0x4DBF, 0x4E00...0x9FFF, 0xAC00...0xD7A3, 0xF900...0xFAFF, 0xFF01...0xFF60, + 0xFFE0...0xFFEF + ] + + public static func sniff(_ data: Data, isWholeFile: Bool = true) -> TabularEncodingSniff { + data.withUnsafeBytes { sniff($0.bindMemory(to: UInt8.self), isWholeFile: isWholeFile) } + } + + public static func sniff(_ bytes: UnsafeBufferPointer, isWholeFile: Bool = true) -> TabularEncodingSniff { + if let marked = byteOrderMarkSniff(bytes) { + return marked + } + let head = UnsafeBufferPointer(rebasing: bytes[0..) -> TabularEncodingSniff? { + for encoding in markedEncodings where hasPrefix(bytes, encoding.byteOrderMark) { + return TabularEncodingSniff(encoding: encoding, byteOrderMarkLength: encoding.byteOrderMark.count) + } + return nil + } + + private static func unmarkedUTF16(_ head: UnsafeBufferPointer) -> TabularTextEncoding? { + let evenLength = head.count - head.count % 2 + guard evenLength >= 4 else { return nil } + var nulBytes = 0 + var nulUnits = 0 + for unit in stride(from: 0, to: evenLength, by: 2) { + let firstIsNul = head[unit] == 0 + let secondIsNul = head[unit + 1] == 0 + nulBytes += (firstIsNul ? 1 : 0) + (secondIsNul ? 1 : 0) + nulUnits += firstIsNul && secondIsNul ? 1 : 0 + } + guard nulBytes * 100 >= evenLength, nulUnits * 100 < evenLength / 2 else { return nil } + let body = UnsafeBufferPointer(rebasing: head[0.. 0, little != big else { return nil } + return little > big ? .utf16LittleEndian : .utf16BigEndian + } + + private static func structuralScalarCount(_ body: UnsafeBufferPointer, as encoding: TabularTextEncoding) -> Int? { + let withoutSplitPair = UnsafeBufferPointer(rebasing: body[0.., isWholeFile: Bool) -> SampleWindow? { + guard let firstNonASCII = firstNonASCIIOffset(in: bytes) else { return nil } + var start = firstNonASCII + let lowestStart = max(0, firstNonASCII - sampleLength / 2) + while start > lowestStart, !isLineBreak(bytes[start - 1]) { + start -= 1 + } + let end = min(bytes.count, start + sampleLength) + guard end < bytes.count || !isWholeFile else { + return SampleWindow(bytes: Data(UnsafeBufferPointer(rebasing: bytes[start.., from lowest: Int, before end: Int) -> Int? { + var lastStandalone: Int? + var offset = end - 1 + while offset >= lowest { + let byte = bytes[offset] + if isLineBreak(byte) { return offset + 1 } + if lastStandalone == nil, byte < TabularCodeUnitLayout.lowestByteInsideACharacter { lastStandalone = offset + 1 } + offset -= 1 + } + return lastStandalone + } + + private static func encoding(of window: SampleWindow) -> TabularTextEncoding { + let isUTF8 = window.bytes.withUnsafeBytes { + isValidUTF8($0.bindMemory(to: UInt8.self), allowingCutTail: window.mayEndInsideCharacter) + } + if isUTF8 || isMostlyUTF8(window.bytes) { return .utf8 } + if let ranked = rankedLegacyEncoding(of: window.bytes), ranked != .windows1252, + holdsEastAsianText(window.bytes, readAs: ranked) { + return ranked + } + return looksLikeHalfWidthKatakana(window.bytes) ? .shiftJIS : .windows1252 + } + + private static func isMostlyUTF8(_ window: Data) -> Bool { + var validLines = 0 + var invalidLines = 0 + for line in window.split(whereSeparator: isLineBreak) where line.contains(where: { $0 >= 0x80 }) { + if String(data: Data(line), encoding: .utf8) == nil { + invalidLines += 1 + } else { + validLines += 1 + } + } + return validLines > invalidLines && invalidLines <= max(1, (validLines + invalidLines) / undecodableLineShareLimit) + } + + private static func holdsEastAsianText(_ window: Data, readAs encoding: TabularTextEncoding) -> Bool { + guard let text = mostlyDecodedText(window, as: encoding) else { return false } + return text.unicodeScalars.contains { scalar in + eastAsianScalars.contains { $0.contains(scalar.value) } + } + } + + private static func mostlyDecodedText(_ window: Data, as encoding: TabularTextEncoding) -> String? { + guard let decoded = try? TabularTextTranscoder.utf8Data(from: window, encoding: encoding, skippingPrefix: 0) else { + return nil + } + let lineCount = max(1, window.reduce(into: 0) { count, byte in + if isLineBreak(byte) { count += 1 } + }) + guard decoded.undecodableLineCount <= max(1, lineCount / undecodableLineShareLimit) else { return nil } + return TabularTextCodec.utf8String(decoded.data) + } + + private static func rankedLegacyEncoding(of window: Data) -> TabularTextEncoding? { + let options: [StringEncodingDetectionOptionsKey: Any] = [ + .suggestedEncodingsKey: legacyCandidates.map { NSNumber(value: $0.foundationEncoding.rawValue) }, + .useOnlySuggestedEncodingsKey: true, + .allowLossyKey: true + ] + let detected = NSString.stringEncoding( + for: window, + encodingOptions: options, + convertedString: nil, + usedLossyConversion: nil + ) + return legacyCandidates.first { $0.foundationEncoding.rawValue == detected } + } + + private static func looksLikeHalfWidthKatakana(_ window: Data) -> Bool { + var nonASCII = 0 + var katakana = 0 + var runs = 0 + var longRuns = 0 + var run = 0 + for byte in window { + if byte >= 0x80 { + nonASCII += 1 + run += 1 + if halfWidthKatakana.contains(byte) { katakana += 1 } + continue + } + if run > 0 { + runs += 1 + if run >= halfWidthKatakanaRunLength { longRuns += 1 } + run = 0 + } + } + if run > 0 { + runs += 1 + if run >= halfWidthKatakanaRunLength { longRuns += 1 } + } + guard nonASCII > 0, runs > 0 else { return false } + let katakanaShare = Double(katakana) / Double(nonASCII) + let longRunShare = Double(longRuns) / Double(runs) + return katakanaShare >= halfWidthKatakanaShare && longRunShare >= halfWidthKatakanaRunShare + && mostlyDecodedText(window, as: .shiftJIS) != nil + } + + private static func firstNonASCIIOffset(in bytes: UnsafeBufferPointer) -> Int? { + guard let base = bytes.baseAddress else { return nil } + let highBits: UInt64 = 0x8080_8080_8080_8080 + let limit = min(bytes.count, asciiScanLimit) + var offset = 0 + while offset + 8 <= limit { + let word = UnsafeRawPointer(base + offset).loadUnaligned(as: UInt64.self) + if word & highBits != 0 { break } + offset += 8 + } + while offset < limit { + if bytes[offset] >= 0x80 { return offset } + offset += 1 + } + return nil + } + + private static func isLineBreak(_ byte: UInt8) -> Bool { + byte == 0x0A || byte == 0x0D + } + + private static func hasPrefix(_ bytes: UnsafeBufferPointer, _ prefix: [UInt8]) -> Bool { + guard !prefix.isEmpty, bytes.count >= prefix.count else { return false } + for (offset, byte) in prefix.enumerated() where bytes[offset] != byte { + return false + } + return true + } + + private static func isValidUTF8(_ bytes: UnsafeBufferPointer, allowingCutTail: Bool) -> Bool { + var end = bytes.count + if allowingCutTail { + var continuation = 0 + while end > 0, continuation < 3, (bytes[end - 1] & 0xC0) == 0x80 { + end -= 1 + continuation += 1 + } + if end > 0, bytes[end - 1] >= 0xC0 { + end -= 1 + } + } + var iterator = UnsafeBufferPointer(rebasing: bytes[0.. String { + guard text.unicodeScalars.contains(where: { $0 == yenSign || $0 == overline }) else { return text } + var scalars = String.UnicodeScalarView() + for scalar in text.unicodeScalars { + switch scalar { + case yenSign: scalars.append("\\") + case overline: scalars.append("~") + default: scalars.append(scalar) + } + } + return String(scalars) + } +} diff --git a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextCodec.swift b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextCodec.swift index 1aeb2e21f2..a4ffa54cff 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextCodec.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextCodec.swift @@ -76,7 +76,9 @@ public enum TabularTextCodec { return Array(string.utf8) case .windows1252, .isoLatin1: return try encodeSingleByte(string, as: encoding) - case .utf16LittleEndian, .utf16BigEndian, .shiftJIS, .gb18030, .big5, .eucKR: + case .shiftJIS, .eucJP: + return try encodeWithFoundation(TabularJISRoman.folded(string), as: encoding) + case .utf16LittleEndian, .utf16BigEndian, .gb18030, .big5, .eucKR: return try encodeWithFoundation(string, as: encoding) } } diff --git a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextEncoding.swift b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextEncoding.swift index d4301500c1..af17177229 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextEncoding.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextEncoding.swift @@ -8,6 +8,7 @@ public enum TabularTextEncoding: String, CaseIterable, Sendable, Codable { case windows1252 case isoLatin1 case shiftJIS + case eucJP case gb18030 case big5 case eucKR @@ -16,7 +17,7 @@ public enum TabularTextEncoding: String, CaseIterable, Sendable, Codable { switch self { case .utf8, .windows1252, .isoLatin1: return true - case .utf16LittleEndian, .utf16BigEndian, .shiftJIS, .gb18030, .big5, .eucKR: + case .utf16LittleEndian, .utf16BigEndian, .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: return false } } @@ -29,11 +30,26 @@ public enum TabularTextEncoding: String, CaseIterable, Sendable, Codable { return [0xFF, 0xFE] case .utf16BigEndian: return [0xFE, 0xFF] - case .windows1252, .isoLatin1, .shiftJIS, .gb18030, .big5, .eucKR: + case .windows1252, .isoLatin1, .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: return [] } } + public var displayName: String { + switch self { + case .utf8: return "UTF-8" + case .utf16LittleEndian: return "UTF-16 LE" + case .utf16BigEndian: return "UTF-16 BE" + case .windows1252: return "Windows-1252" + case .isoLatin1: return "ISO Latin 1" + case .shiftJIS: return "Shift JIS" + case .eucJP: return "EUC-JP" + case .gb18030: return "GB 18030" + case .big5: return "Big5" + case .eucKR: return "EUC-KR" + } + } + public var foundationEncoding: String.Encoding { switch self { case .utf8: @@ -48,12 +64,25 @@ public enum TabularTextEncoding: String, CaseIterable, Sendable, Codable { return .isoLatin1 case .shiftJIS: return .shiftJIS + case .eucJP: + return .japaneseEUC case .gb18030: return Self.coreFoundationEncoding(CFStringEncodings.GB_18030_2000) case .big5: - return Self.coreFoundationEncoding(CFStringEncodings.big5) + return Self.coreFoundationEncoding(CFStringEncodings.dosChineseTrad) case .eucKR: - return Self.coreFoundationEncoding(CFStringEncodings.EUC_KR) + return Self.coreFoundationEncoding(CFStringEncodings.dosKorean) + } + } + + var codeUnitLayout: TabularCodeUnitLayout { + switch self { + case .utf16LittleEndian: + return .utf16LittleEndian + case .utf16BigEndian: + return .utf16BigEndian + case .utf8, .windows1252, .isoLatin1, .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: + return .asciiCompatible } } diff --git a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextTranscoder.swift b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextTranscoder.swift index f0ea615c1e..5362307de0 100644 --- a/Packages/TableProCore/Sources/TableProTabularIO/TabularTextTranscoder.swift +++ b/Packages/TableProCore/Sources/TableProTabularIO/TabularTextTranscoder.swift @@ -1,20 +1,350 @@ import Foundation -public enum TabularTranscodingError: Error, Equatable, Sendable { - case undecodable(TabularTextEncoding) +public struct TabularTranscodedText: Sendable { + public let map: TabularTranscodingMap + public let undecodableLineCount: Int + public let firstUndecodableLine: Int? +} + +public struct TabularTranscodedData: Sendable { + public let data: Data + public let undecodableLineCount: Int + public let firstUndecodableLine: Int? } public enum TabularTextTranscoder { + public static let chunkLength = 1 << 20 + + private static let replacementCharacter: [UInt8] = Array("\u{FFFD}".utf8) + private static let longestCharacterLength = 4 + + public static func transcode( + _ data: Data, + from encoding: TabularTextEncoding, + skippingPrefix prefixLength: Int, + to url: URL, + progress: (Double) -> Void = { _ in }, + isCancelled: () -> Bool = { false } + ) throws -> TabularTranscodedText { + guard FileManager.default.createFile(atPath: url.path, contents: nil) else { + throw TabularWriteError.couldNotCreate(path: url.path) + } + let handle: FileHandle + do { + handle = try FileHandle(forWritingTo: url) + } catch { + throw TabularWriteError.writeFailed(message: error.localizedDescription) + } + defer { try? handle.close() } + let outcome = try run(data, from: encoding, skippingPrefix: prefixLength, progress: progress, isCancelled: isCancelled) { + do { + try handle.write(contentsOf: $0) + } catch { + throw TabularWriteError.writeFailed(message: error.localizedDescription) + } + } + let map = TabularTranscodingMap( + original: data, + encoding: encoding, + transcodedStarts: outcome.transcodedStarts, + originalStarts: outcome.originalStarts + ) + return TabularTranscodedText( + map: map, + undecodableLineCount: outcome.undecodableLineCount, + firstUndecodableLine: outcome.firstUndecodableLine + ) + } + public static func utf8Data( from data: Data, encoding: TabularTextEncoding, - skippingPrefix prefixLength: Int - ) throws -> Data { - let body = data.dropFirst(min(prefixLength, data.count)) - guard encoding != .utf8 else { return Data(body) } - guard let text = String(data: Data(body), encoding: encoding.foundationEncoding) else { - throw TabularTranscodingError.undecodable(encoding) - } - return Data(text.utf8) + skippingPrefix prefixLength: Int, + wholeLinesWithin length: Int = .max + ) throws -> TabularTranscodedData { + let end = wholeLinesEnd(of: data, encoding: encoding, from: prefixLength, within: length) + var output = Data() + let outcome = try run(data.prefix(end), from: encoding, skippingPrefix: prefixLength, progress: { _ in }, isCancelled: { false }) { + output.append(contentsOf: $0) + } + return TabularTranscodedData( + data: output, + undecodableLineCount: outcome.undecodableLineCount, + firstUndecodableLine: outcome.firstUndecodableLine + ) + } + + private static func wholeLinesEnd(of data: Data, encoding: TabularTextEncoding, from start: Int, within length: Int) -> Int { + let start = min(start, data.count) + guard length < data.count - start else { return data.count } + return data.withUnsafeBytes { raw in + let bytes = raw.bindMemory(to: UInt8.self) + return encoding.codeUnitLayout.endOfLastLineBreak(in: bytes, from: start, before: start + length) ?? start + length + } + } + + public static func firstInvalidUTF8Line(in data: Data, skippingPrefix prefixLength: Int) -> Int? { + data.withUnsafeBytes { raw in + let bytes = raw.bindMemory(to: UInt8.self) + let layout = TabularCodeUnitLayout.asciiCompatible + var lineStart = min(prefixLength, bytes.count) + var lineNumber = 1 + while lineStart < bytes.count { + let lineEnd = layout.endOfNextLineBreak(in: bytes, from: lineStart, notBefore: lineStart) ?? bytes.count + let line = UnsafeBufferPointer(rebasing: bytes[lineStart..= 0x80 }), !isValidUTF8(line) { + return lineNumber + } + lineNumber += 1 + lineStart = lineEnd + } + return nil + } + } + + private static func isValidUTF8(_ bytes: UnsafeBufferPointer) -> Bool { + var iterator = bytes.makeIterator() + var decoder = UTF8() + while true { + switch decoder.decode(&iterator) { + case .scalarValue: + continue + case .emptyInput: + return true + case .error: + return false + } + } + } + + private struct Outcome { + var transcodedStarts: [Int] = [] + var originalStarts: [Int] = [] + var undecodableLineCount = 0 + var firstUndecodableLine: Int? + } + + private struct DecodedChunk { + var utf8: [UInt8] = [] + var undecodableLineCount = 0 + var firstUndecodableLineIndex: Int? + var startsUndecodable = false + var endsUndecodable = false + } + + private static func run( + _ data: Data, + from encoding: TabularTextEncoding, + skippingPrefix prefixLength: Int, + progress: (Double) -> Void, + isCancelled: () -> Bool, + emit: ([UInt8]) throws -> Void + ) throws -> Outcome { + let layout = encoding.codeUnitLayout + var outcome = Outcome() + try data.withUnsafeBytes { raw in + let bytes = raw.bindMemory(to: UInt8.self) + var start = min(prefixLength, bytes.count) + var transcodedOffset = 0 + var linesBefore = 0 + var openLineIsCounted = false + outcome.transcodedStarts.append(0) + outcome.originalStarts.append(start) + while start < bytes.count { + if isCancelled() { throw TabularCancellation() } + let end = chunkEnd(in: bytes, from: start, layout: layout) + if start > outcome.originalStarts[outcome.originalStarts.count - 1] { + outcome.transcodedStarts.append(transcodedOffset) + outcome.originalStarts.append(start) + } + let chunk = UnsafeBufferPointer(rebasing: bytes[start.., from start: Int, layout: TabularCodeUnitLayout) -> Int { + let target = start + chunkLength + guard target < bytes.count else { return bytes.count } + return layout.endOfLastLineBreak(in: bytes, from: start, before: target) + ?? layout.endOfLastCharacter(in: bytes, from: start, before: target) + ?? layout.endOfNextCharacter(in: bytes, from: start, notBefore: target) + ?? bytes.count + } + + private static func decode(_ chunk: UnsafeBufferPointer, encoding: TabularTextEncoding) -> DecodedChunk { + switch encoding { + case .utf8: + return DecodedChunk(utf8: Array(chunk)) + case .windows1252, .isoLatin1: + var utf8: [UInt8] = [] + utf8.reserveCapacity(chunk.count + chunk.count / 4) + TabularTextCodec.appendUTF8(of: chunk, from: encoding, into: &utf8) + return DecodedChunk(utf8: utf8) + case .utf16LittleEndian, .utf16BigEndian, .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: + let layout = encoding.codeUnitLayout + if let strict = strictUTF8(chunk, encoding: encoding), + layout.lineBreakCount(in: chunk, from: 0, to: chunk.count) + == strict.withUnsafeBufferPointer({ TabularCodeUnitLayout.asciiCompatible.lineBreakCount(in: $0, from: 0, to: $0.count) }) { + return DecodedChunk(utf8: strict) + } + return decodeLineByLine(chunk, encoding: encoding) + } + } + + private static func strictUTF8(_ bytes: UnsafeBufferPointer, encoding: TabularTextEncoding) -> [UInt8]? { + guard bytes.count.isMultiple(of: encoding.codeUnitLayout.width) else { return nil } + guard !bytes.isEmpty else { return [] } + guard isWellFormed(bytes, as: encoding), + let text = String(bytes: bytes, encoding: encoding.foundationEncoding) else { return nil } + return Array(text.utf8) + } + + private static func isWellFormed(_ bytes: UnsafeBufferPointer, as encoding: TabularTextEncoding) -> Bool { + guard encoding == .eucJP else { return true } + var offset = 0 + while offset < bytes.count { + guard let length = eucJPCharacterLength(in: bytes, at: offset) else { return false } + offset += length + } + return true + } + + private static func eucJPCharacterLength(in bytes: UnsafeBufferPointer, at offset: Int) -> Int? { + switch bytes[offset] { + case 0x00...0x7F: + return 1 + case 0x8E: + return hasTrail(bytes, after: offset, count: 1, in: 0xA1...0xDF) ? 2 : nil + case 0x8F: + return hasTrail(bytes, after: offset, count: 2, in: 0xA1...0xFE) ? 3 : nil + case 0xA1...0xFE: + return hasTrail(bytes, after: offset, count: 1, in: 0xA1...0xFE) ? 2 : nil + default: + return nil + } + } + + private static func hasTrail( + _ bytes: UnsafeBufferPointer, + after offset: Int, + count: Int, + in range: ClosedRange + ) -> Bool { + guard offset + count < bytes.count else { return false } + return (1...count).allSatisfy { range.contains(bytes[offset + $0]) } + } + + private static func decodeLineByLine(_ chunk: UnsafeBufferPointer, encoding: TabularTextEncoding) -> DecodedChunk { + let layout = encoding.codeUnitLayout + var decoded = DecodedChunk() + decoded.utf8.reserveCapacity(chunk.count * 2) + var lineStart = 0 + var offset = 0 + while offset + layout.width <= chunk.count { + guard let lineBreak = layout.lineBreakByte(in: chunk, at: offset) else { + offset += layout.width + continue + } + appendLine(in: chunk, lineStart.., + _ range: Range, + encoding: TabularTextEncoding, + into decoded: inout DecodedChunk + ) { + let line = UnsafeBufferPointer(rebasing: chunk[range]) + if let strict = strictUTF8(line, encoding: encoding) { + decoded.utf8.append(contentsOf: strict) + return + } + decoded.undecodableLineCount += 1 + decoded.startsUndecodable = decoded.startsUndecodable || range.lowerBound == 0 + decoded.endsUndecodable = range.upperBound == chunk.count + if decoded.firstUndecodableLineIndex == nil { + decoded.firstUndecodableLineIndex = encoding.codeUnitLayout.lineCount(in: chunk, from: 0, to: range.lowerBound) + } + switch encoding.codeUnitLayout { + case .utf16LittleEndian, .utf16BigEndian: + appendLossyUTF16(line, layout: encoding.codeUnitLayout, into: &decoded.utf8) + case .asciiCompatible: + appendLossyMultiByte(line, encoding: encoding, into: &decoded.utf8) + } + } + + private static func appendLossyUTF16(_ line: UnsafeBufferPointer, layout: TabularCodeUnitLayout, into utf8: inout [UInt8]) { + let unitCount = line.count / 2 + var units: [UInt16] = [] + units.reserveCapacity(unitCount) + for index in 0.., encoding: TabularTextEncoding, into utf8: inout [UInt8]) { + var offset = 0 + while offset < line.count { + let byte = line[offset] + if byte < 0x80 { + utf8.append(byte) + offset += 1 + continue + } + guard let character = firstCharacter(in: line, at: offset, encoding: encoding) else { + utf8.append(contentsOf: replacementCharacter) + offset += 1 + continue + } + utf8.append(contentsOf: character.text.utf8) + offset += character.length + } + } + + private static func firstCharacter( + in line: UnsafeBufferPointer, + at offset: Int, + encoding: TabularTextEncoding + ) -> (text: String, length: Int)? { + let longest = min(longestCharacterLength, line.count - offset) + for length in 1...longest { + let candidate = UnsafeBufferPointer(rebasing: line[offset.. [Int] { + transcoded.withUnsafeBytes { transcodedRaw in + original.withUnsafeBytes { originalRaw in + originalOffsets( + of: ascendingOffsets, + transcoded: transcodedRaw.bindMemory(to: UInt8.self), + original: originalRaw.bindMemory(to: UInt8.self) + ) + } + } + } + + private func originalOffsets( + of ascendingOffsets: [Int], + transcoded: UnsafeBufferPointer, + original originalBytes: UnsafeBufferPointer + ) -> [Int] { + let layout = encoding.codeUnitLayout + var checkpoint = 0 + var transcodedPosition = transcodedStarts[0] + var originalPosition = originalStarts[0] + var offsets: [Int] = [] + offsets.reserveCapacity(ascendingOffsets.count) + for target in ascendingOffsets { + guard target < transcoded.count else { + offsets.append(originalBytes.count) + continue + } + while checkpoint + 1 < transcodedStarts.count, transcodedStarts[checkpoint + 1] <= target { + checkpoint += 1 + transcodedPosition = transcodedStarts[checkpoint] + originalPosition = originalStarts[checkpoint] + } + let lineBreaks = TabularCodeUnitLayout.asciiCompatible.lineBreakCount( + in: transcoded, + from: transcodedPosition, + to: target + ) + originalPosition = layout.offset(afterLineBreaks: lineBreaks, in: originalBytes, from: originalPosition) + transcodedPosition = target + offsets.append(originalPosition) + } + return offsets + } +} diff --git a/Packages/TableProCore/Tests/TableProTabularIOTests/TabularEncodingDetectorTests.swift b/Packages/TableProCore/Tests/TableProTabularIOTests/TabularEncodingDetectorTests.swift new file mode 100644 index 0000000000..a2ddcf5cb2 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProTabularIOTests/TabularEncodingDetectorTests.swift @@ -0,0 +1,212 @@ +import CoreFoundation +import Foundation +@testable import TableProTabularIO +import XCTest + +final class TabularEncodingDetectorTests: XCTestCase { + private let japaneseRows = """ + 商品コード,商品名,在庫管理区分,ｻﾞｲｺｶﾝﾘｸﾌﾞﾝ,備考 + 10001,ソフトウェア保守,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,表示のみ + 10002,ポイント交換,在庫管理しない,ｻﾞｲｺｶﾝﾘｼﾅｲ,ー + 10003,髙橋商店の注文,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,①から③まで + 10004,東京都港区芝公園４－２－８,在庫管理しない,ｻﾞｲｺｶﾝﾘｼﾅｲ,～ + 10005,山﨑さんの見積書,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,確認済み + + """ + + private func encoded(_ text: String, _ encoding: TabularTextEncoding) throws -> Data { + try XCTUnwrap(text.data(using: encoding.foundationEncoding, allowLossyConversion: false)) + } + + private func sniff(_ data: Data, isWholeFile: Bool = true) -> TabularEncodingSniff { + TabularEncodingDetector.sniff(data, isWholeFile: isWholeFile) + } + + func testByteOrderMarksDecideTheEncoding() { + XCTAssertEqual(sniff(Data([0xEF, 0xBB, 0xBF, 0x61])), TabularEncodingSniff(encoding: .utf8, byteOrderMarkLength: 3)) + XCTAssertEqual(sniff(Data([0xFF, 0xFE, 0x61, 0x00])), TabularEncodingSniff(encoding: .utf16LittleEndian, byteOrderMarkLength: 2)) + XCTAssertEqual(sniff(Data([0xFE, 0xFF, 0x00, 0x61])), TabularEncodingSniff(encoding: .utf16BigEndian, byteOrderMarkLength: 2)) + } + + func testASCIIAndUTF8ReadAsUTF8() { + XCTAssertEqual(sniff(Data("id,name\n1,Ann\n".utf8)).encoding, .utf8) + XCTAssertEqual(sniff(Data(japaneseRows.utf8)).encoding, .utf8) + XCTAssertEqual(sniff(Data()).encoding, .utf8) + } + + func testShiftJISExportIsDetected() throws { + XCTAssertEqual(sniff(try encoded(japaneseRows, .shiftJIS)).encoding, .shiftJIS) + } + + func testHalfWidthKatakanaOnlyReadsAsShiftJIS() throws { + let screenshotHeader: [UInt8] = [0xBB, 0xDE, 0xB2, 0xBA, 0xB6, 0xDD, 0xD8, 0xB8, 0xCC, 0xDE, 0xDD] + var bytes = Array("CODE,".utf8) + screenshotHeader + [0x0A] + let rows = "1,ｻﾞｲｺｶﾝﾘｽﾙ\n2,ﾁｮｳﾘｱﾘ\n3,ｿﾌﾄｳｪｱ\n4,ﾎﾟｲﾝﾄ\n" + bytes += [UInt8](try encoded(rows, .shiftJIS)) + XCTAssertEqual(sniff(Data(bytes)).encoding, .shiftJIS) + } + + func testTrailBytesThatSpellBackslashAndPipeStayShiftJIS() throws { + let text = "名前|読み\nソフト|ｿﾌﾄ\n表計算|ﾋｮｳｹｲｻﾝ\nポイント|ﾎﾟｲﾝﾄ\n" + XCTAssertEqual(sniff(try encoded(text, .shiftJIS)).encoding, .shiftJIS) + } + + func testABadByteInTheSampleDoesNotHideTheEncoding() throws { + var bytes = try encoded(japaneseRows, .shiftJIS) + bytes.append(contentsOf: Array("10006,".utf8) + [0xA0] + Array(",x\n".utf8)) + bytes.append(try encoded(japaneseRows, .shiftJIS)) + XCTAssertEqual(sniff(bytes).encoding, .shiftJIS) + + var halfWidth = try encoded(String(repeating: "1,ｻﾞｲｺｶﾝﾘｽﾙ,ﾁｮｳﾘｱﾘ\n", count: 60), .shiftJIS) + halfWidth.append(contentsOf: Array("2,".utf8) + [0xA0] + [0x0A]) + XCTAssertEqual(sniff(halfWidth).encoding, .shiftJIS) + } + + func testEUCJPIsDetected() throws { + let text = """ + 商品コード,商品名,在庫管理区分,ｻﾞｲｺｶﾝﾘｸﾌﾞﾝ,備考 + 10001,ソフトウェア保守,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,表示のみ + 10002,ポイント交換,在庫管理しない,ｻﾞｲｺｶﾝﾘｼﾅｲ,確認中 + 10003,高橋商店の注文,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,一から三まで + 10004,東京都港区芝公園,在庫管理しない,ｻﾞｲｺｶﾝﾘｼﾅｲ,発送済み + 10005,山崎さんの見積書,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ,確認済み + + """ + XCTAssertEqual(sniff(try encoded(text, .eucJP)).encoding, .eucJP) + } + + func testSimplifiedChineseIsDetectedAsGB18030() throws { + let text = """ + 客户编号,客户名称,所在城市,备注 + 1001,北京科技有限公司,北京市朝阳区,重要客户 + 1002,上海贸易集团,上海市浦东新区,需要跟进 + 1003,广州电子商务中心,广州市天河区,已经签约 + + """ + XCTAssertEqual(sniff(try encoded(text, .gb18030)).encoding, .gb18030) + } + + func testTraditionalChineseWithWindowsOnlyCharactersIsDetectedAsBig5() throws { + let text = """ + 客戶編號,客戶名稱,所在城市,備註 + 2001,臺北科技股份有限公司,臺北市信義區,碁盤裏的重要客戶 + 2002,高雄貿易集團,高雄市前鎮區,需要追蹤 + 2003,臺中電子商務中心,臺中市西屯區,已經簽約 + + """ + let bytes = try encoded(text, .big5) + XCTAssertNil(String(data: bytes, encoding: Self.strictBig5), "the sample holds cp950-only characters") + XCTAssertEqual(sniff(bytes).encoding, .big5) + } + + func testKoreanWithUnifiedHangulIsDetectedAsEUCKR() throws { + let text = """ + 고객번호,고객명,도시,비고 + 3001,똠방각하 무역,서울특별시 강남구,중요 고객 + 3002,한국전자 주식회사,부산광역시 해운대구,후속 조치 필요 + 3003,뷁 상사,대구광역시 수성구,계약 완료 + + """ + XCTAssertEqual(sniff(try encoded(text, .eucKR)).encoding, .eucKR) + } + + func testWesternTextStaysWindows1252() throws { + let text = """ + Kunde;Ort;Straße;Notiz + Müller GmbH;München;Hauptstraße 12;“Wichtig” – 5 € + BÄCKEREI SCHRÖDER;KÖLN;DÜSSELDORFER STR. 5;DEUTSCHLAND + Société Générale;Genève;Rue du Rhône;Crème brûlée + + """ + XCTAssertEqual(sniff(try encoded(text, .windows1252)).encoding, .windows1252) + } + + func testShortWesternFilesWithSymbolsAndCapitalsStayWindows1252() throws { + let samples = [ + "Book,£12.50\nMusic,£9.99\n", + "Item;Price\nBook;£12.50\nFood;€5.00\n", + "Name,Note\nACME ©2024,Registered ®\n", + "SKU1,Item 1,£1.99\n", + "S1,5 µm\nx,½ in\nx,10 m²\ntemp,20°C\n", + "KUNDE;ORT\nMÜLLER GMBH;MÜNCHEN\nBÄCKEREI SCHRÖDER;KÖLN\n", + "NOM;VILLE\nSOCIÉTÉ GÉNÉRALE;GENÈVE\nHÉLÈNE;ORLÉANS\n", + "id,amount\n1,0\u{A0}234,56 €\n2,1\u{A0}000,00 €\n", + "id,amount\n1,£5\n2,¥300\n3,€7\n4,¢9\n" + ] + for sample in samples { + XCTAssertEqual(sniff(try encoded(sample, .windows1252)).encoding, .windows1252, sample) + } + } + + func testMostlyUTF8WithAStrayByteStaysUTF8() { + var bytes = Data("name;city\nJürgen Müller;Köln\nStraße;Düsseldorf\n„Zitat“;Größe\n".utf8) + bytes.append(contentsOf: Array("caf".utf8) + [0xE9] + Array(";x\n".utf8)) + XCTAssertEqual(sniff(bytes).encoding, .utf8) + } + + func testAGB18030CharacterAcrossTheSampleEdgeIsNotCut() throws { + let fourByteCharacter: [UInt8] = [0x94, 0x39, 0xFC, 0x36] + XCTAssertNotNil(String(bytes: fourByteCharacter, encoding: TabularTextEncoding.gb18030.foundationEncoding)) + let row = try encoded("北京市朝阳区建国路八十八号", .gb18030) + for shift in 0..<4 { + var bytes = Data() + while bytes.count < TabularEncodingDetector.sampleLength - shift - 2 { + bytes.append(row) + } + bytes.append(contentsOf: fourByteCharacter) + bytes.append(row) + bytes.append(0x0A) + XCTAssertEqual(sniff(bytes).encoding, .gb18030, "shift \(shift)") + } + } + + func testUnmarkedUTF16CJKWithZeroLowBytesIsDetected() throws { + let texts = [ + String(repeating: "1000,山田\u{3000}太郎,東京都港区芝公園4-2-8,03-1234-5000\n", count: 30), + String(repeating: "一一一一一一一一一一一一一一一一,一\n", count: 20), + String(repeating: String(repeating: "漢字", count: 26) + "," + String(repeating: "漢字", count: 12) + "\n", count: 20) + ] + for text in texts { + XCTAssertEqual(sniff(try encoded(text, .utf16LittleEndian)).encoding, .utf16LittleEndian) + XCTAssertEqual(sniff(try encoded(text, .utf16BigEndian)).encoding, .utf16BigEndian) + } + } + + func testAFileEndingInAWesternByteIsNotReadAsUTF8() { + XCTAssertEqual(sniff(Data([0x6E, 0x0A, 0x43, 0x61, 0x66, 0xE9])).encoding, .windows1252) + } + + func testUTF16WithoutAByteOrderMarkIsDetected() throws { + let text = "名前,住所\n吉田,三上\n本田,日本\nAnn,Paris\n" + XCTAssertEqual(sniff(try encoded(text, .utf16LittleEndian)), TabularEncodingSniff(encoding: .utf16LittleEndian, byteOrderMarkLength: 0)) + XCTAssertEqual(sniff(try encoded(text, .utf16BigEndian)), TabularEncodingSniff(encoding: .utf16BigEndian, byteOrderMarkLength: 0)) + } + + func testShiftJISAfterALongASCIIPrefixIsDetected() throws { + var bytes = Data(String(repeating: "12345,ABCDE,2024-01-01\n", count: 14_000).utf8) + XCTAssertGreaterThan(bytes.count, 300_000) + bytes.append(try encoded(japaneseRows, .shiftJIS)) + XCTAssertEqual(sniff(bytes).encoding, .shiftJIS) + } + + func testCarriageReturnLinesAreDetected() throws { + let text = japaneseRows.replacingOccurrences(of: "\n", with: "\r") + XCTAssertEqual(sniff(try encoded(text, .shiftJIS)).encoding, .shiftJIS) + } + + func testAPrefixCutInsideACharacterStaysUTF8() { + let whole = Data(String(repeating: "東京都港区,", count: 20_000).utf8) + let cut = whole.prefix(TabularEncodingDetector.sampleLength + 1) + XCTAssertEqual(sniff(Data(cut), isWholeFile: false).encoding, .utf8) + } + + func testEveryCandidateIsAnEncodingFoundationKnows() { + for candidate in TabularEncodingDetector.legacyCandidates { + XCTAssertNotNil(String(data: Data("a".utf8), encoding: candidate.foundationEncoding), "\(candidate)") + } + } + + private static let strictBig5 = String.Encoding( + rawValue: CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringEncodings.big5.rawValue)) + ) +} diff --git a/Packages/TableProCore/Tests/TableProTabularIOTests/TabularTextTranscoderTests.swift b/Packages/TableProCore/Tests/TableProTabularIOTests/TabularTextTranscoderTests.swift new file mode 100644 index 0000000000..b053f05542 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProTabularIOTests/TabularTextTranscoderTests.swift @@ -0,0 +1,149 @@ +import Foundation +@testable import TableProTabularIO +import XCTest + +final class TabularTextTranscoderTests: XCTestCase { + private var directory: URL! + + override func setUpWithError() throws { + directory = FileManager.default.temporaryDirectory.appendingPathComponent("TabularTextTranscoderTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: directory) + } + + private func encoded(_ text: String, _ encoding: TabularTextEncoding) throws -> Data { + try XCTUnwrap(text.data(using: encoding.foundationEncoding, allowLossyConversion: false)) + } + + private func transcode(_ data: Data, _ encoding: TabularTextEncoding, prefix: Int = 0) throws -> (String, TabularTranscodedText) { + let url = directory.appendingPathComponent(UUID().uuidString) + let result = try TabularTextTranscoder.transcode(data, from: encoding, skippingPrefix: prefix, to: url) + return (TabularTextCodec.utf8String(try Data(contentsOf: url)), result) + } + + func testShiftJISBecomesTheSameTextInUTF8() throws { + let text = "id,名前\r\n1,髙橋 ①\r\n2,ｻﾞｲｺ ～\r\n" + let (output, result) = try transcode(try encoded(text, .shiftJIS), .shiftJIS) + XCTAssertEqual(output, text) + XCTAssertEqual(result.undecodableLineCount, 0) + } + + func testABadByteSpoilsOnlyItsOwnLine() throws { + var bytes = try encoded("a,髙橋\n", .shiftJIS) + bytes.append(contentsOf: Array("b,".utf8) + [0x82] + Array("\nc,".utf8) + [0xA0] + Array("x\n".utf8)) + bytes.append(try encoded("d,東京\ne,大阪\n", .shiftJIS)) + let (output, result) = try transcode(bytes, .shiftJIS) + XCTAssertEqual(output, "a,髙橋\nb,\u{FFFD}\nc,\u{FFFD}x\nd,東京\ne,大阪\n") + XCTAssertEqual(result.undecodableLineCount, 2) + } + + func testLinesSplitAcrossChunksMatchAWholeFileDecode() throws { + let line = "10001,東京都港区芝公園４－２－８,ｻﾞｲｺｶﾝﾘｽﾙ,髙橋\r\n" + let text = String(repeating: line, count: 60_000) + let bytes = try encoded(text, .shiftJIS) + XCTAssertGreaterThan(bytes.count, TabularTextTranscoder.chunkLength * 2) + let (output, result) = try transcode(bytes, .shiftJIS) + XCTAssertEqual(output, text) + XCTAssertGreaterThan(result.map.transcodedStarts.count, 2) + XCTAssertEqual(result.map.transcodedStarts.count, result.map.originalStarts.count) + } + + func testALineLongerThanAChunkIsCutAtACharacterBoundary() throws { + let field = String(repeating: "東京都港区芝公園４－２－８ ", count: 90_000) + let text = "id,memo\n1," + field + "\n2,x\n" + let bytes = try encoded(text, .shiftJIS) + XCTAssertGreaterThan(bytes.count, TabularTextTranscoder.chunkLength * 2) + let (output, result) = try transcode(bytes, .shiftJIS) + XCTAssertEqual(output, text) + XCTAssertEqual(result.undecodableLineCount, 0) + XCTAssertGreaterThan(result.map.transcodedStarts.count, 2) + } + + func testABrokenLineLongerThanAChunkIsCountedOnce() throws { + let piece = try encoded(String(repeating: "東京都港区芝公園 ", count: 5_000), .shiftJIS) + var bytes = try encoded("id,memo\n1,", .shiftJIS) + for _ in 0..<30 { + bytes.append(piece) + bytes.append(0xA0) + } + bytes.append(contentsOf: Array("\n2,x\n".utf8)) + XCTAssertGreaterThan(bytes.count, TabularTextTranscoder.chunkLength * 2) + let (output, result) = try transcode(bytes, .shiftJIS) + XCTAssertEqual(result.undecodableLineCount, 1) + XCTAssertEqual(result.firstUndecodableLine, 2) + XCTAssertTrue(output.hasSuffix("\n2,x\n")) + } + + func testAMalformedEUCJPByteKeepsTheDelimiterAfterIt() throws { + let bytes = Data(Array("1,".utf8) + [0x8E] + Array(",x\n\"a".utf8) + [0xA4] + Array("\",b\n".utf8)) + let (output, result) = try transcode(bytes, .eucJP) + XCTAssertEqual(output, "1,\u{FFFD},x\n\"a\u{FFFD}\",b\n") + XCTAssertEqual(result.undecodableLineCount, 2) + } + + func testTheFirstUndecodableLineIsNumberedFromOne() throws { + var bytes = try encoded("a\r\nb\r\nc\r\n", .shiftJIS) + bytes.append(contentsOf: [0x64, 0xA0, 0x0D, 0x0A]) + let (_, result) = try transcode(bytes, .shiftJIS) + XCTAssertEqual(result.firstUndecodableLine, 4) + XCTAssertNil(try transcode(try encoded("a\nb\n", .shiftJIS), .shiftJIS).1.firstUndecodableLine) + } + + func testTheFirstInvalidUTF8LineIsFound() { + let bytes = Data("a,b\nMüller,1\n".utf8) + Data([0x43, 0x61, 0x66, 0xE9, 0x0A]) + Data("z\n".utf8) + XCTAssertEqual(TabularTextTranscoder.firstInvalidUTF8Line(in: bytes, skippingPrefix: 0), 3) + XCTAssertNil(TabularTextTranscoder.firstInvalidUTF8Line(in: Data("a\nÜ\n".utf8), skippingPrefix: 0)) + } + + func testCarriageReturnOnlyLinesKeepTheirBreaks() throws { + let text = "a,日本\rb,中文\rc,한국\r" + let (output, _) = try transcode(try encoded(text, .utf16LittleEndian), .utf16LittleEndian) + XCTAssertEqual(output, text) + } + + func testByteOrderMarkIsSkipped() throws { + let text = "a,本\n" + let bytes = Data([0xFF, 0xFE]) + (try encoded(text, .utf16LittleEndian)) + let (output, result) = try transcode(bytes, .utf16LittleEndian, prefix: 2) + XCTAssertEqual(output, text) + XCTAssertEqual(result.map.originalStarts.first, 2) + } + + func testAnOddTrailingUTF16ByteBecomesAReplacementCharacter() throws { + let bytes = (try encoded("a,b\n日本", .utf16LittleEndian)) + Data([0x41]) + let (output, result) = try transcode(bytes, .utf16LittleEndian) + XCTAssertEqual(output, "a,b\n日本\u{FFFD}") + XCTAssertEqual(result.undecodableLineCount, 1) + } + + func testWindows1252UsesTheCodecsOwnTable() throws { + let (output, result) = try transcode(Data([0x43, 0x61, 0x66, 0xE9, 0x20, 0x80, 0x81, 0x0A]), .windows1252) + XCTAssertEqual(output, "Café €\u{81}\n") + XCTAssertEqual(result.undecodableLineCount, 0) + } + + func testAPrefixEndsOnTheLastWholeLine() throws { + let text = "a,日本\nb,中文\nc,한국\n" + let bytes = try encoded(text, .utf16BigEndian) + let firstTwoLines = try encoded("a,日本\nb,中文\n", .utf16BigEndian).count + let prefix = try TabularTextTranscoder.utf8Data( + from: bytes, + encoding: .utf16BigEndian, + skippingPrefix: 0, + wholeLinesWithin: firstTwoLines + 3 + ) + XCTAssertEqual(TabularTextCodec.utf8String(prefix.data), "a,日本\nb,中文\n") + XCTAssertEqual(prefix.undecodableLineCount, 0) + } + + func testCancellationStopsTheTranscode() throws { + let bytes = try encoded(String(repeating: "東京,大阪\n", count: 200_000), .shiftJIS) + let url = directory.appendingPathComponent("cancelled") + XCTAssertThrowsError(try TabularTextTranscoder.transcode(bytes, from: .shiftJIS, skippingPrefix: 0, to: url, isCancelled: { true })) { + XCTAssertTrue($0 is TabularCancellation) + } + } +} diff --git a/Packages/TableProCore/Tests/TableProTabularTests/TabularSaveTests.swift b/Packages/TableProCore/Tests/TableProTabularTests/TabularSaveTests.swift index 34a78d2eb7..6e8b16936e 100644 --- a/Packages/TableProCore/Tests/TableProTabularTests/TabularSaveTests.swift +++ b/Packages/TableProCore/Tests/TableProTabularTests/TabularSaveTests.swift @@ -16,25 +16,13 @@ final class TabularSaveTests: XCTestCase { } private func open(_ bytes: [UInt8], fileExtension: String = "csv") async throws -> (DelimitedSource, TabularTable) { - let data = Data(bytes) - let dialect = data.withUnsafeBytes { raw -> DelimitedDialect in - let buffer = raw.bindMemory(to: UInt8.self) - let sniff = DelimitedDialectDetector.sniffEncoding(buffer) - return DelimitedDialectDetector.detect( - buffer, - contentStart: sniff.byteOrderMarkLength, - encoding: sniff.encoding, - hasByteOrderMark: sniff.hasByteOrderMark, - fileExtension: fileExtension - ) - } - let source = try await DelimitedSourceBuilder.build( - bytes: data, - dialect: dialect, - byteEncoding: dialect.encoding, - contentStart: dialect.hasByteOrderMark ? dialect.encoding.byteOrderMark.count : 0 + let contents = try await DelimitedFileReader.read( + Data(bytes), + fileExtension: fileExtension, + transcodedFileURL: directory.appendingPathComponent("utf8-\(UUID().uuidString)") ) - return (source, TabularTable(source: source, usesFirstRowAsHeader: dialect.hasHeaderRow)) + let source = contents.source + return (source, TabularTable(source: source, usesFirstRowAsHeader: source.dialect.hasHeaderRow)) } private func save(_ table: TabularTable, source: DelimitedSource, dialect: DelimitedDialect? = nil) throws -> [UInt8] { @@ -138,4 +126,127 @@ final class TabularSaveTests: XCTestCase { let (tieSource, _) = try await open(Array("a,b;c\n".utf8)) XCTAssertEqual(tieSource.dialect.delimiter, DelimitedDialect.comma) } + + private func shiftJIS(_ text: String) throws -> [UInt8] { + [UInt8](try XCTUnwrap(text.data(using: .shiftJIS, allowLossyConversion: false))) + } + + private func quotedShiftJISExport() throws -> (bytes: [UInt8], rows: [[UInt8]]) { + let taka: [UInt8] = [0xFB, 0xFC] + let saki: [UInt8] = [0xFA, 0xB1] + let smallRomanOne: [UInt8] = [0xFA, 0x40] + let rows: [[UInt8]] = [ + try shiftJIS("\"顧客ID\",\"氏名\",\"よみがな\",\"備考\"\r\n"), + try shiftJIS("\"001\",\"") + taka + (try shiftJIS("橋\",\"たかはし\",\"ｿﾌﾄｳｪｱの保守\"\r\n")), + try shiftJIS("\"002\",\"山") + saki + (try shiftJIS("\",\"やまさき\",\"")) + smallRomanOne + (try shiftJIS("の注文\"\r\n")), + try shiftJIS("\"003\",\"田中\",\"たなか\",\"ポイント交換\"\r\n"), + try shiftJIS("\"004\",\"佐藤\",\"さとう\",\"東京都港区芝公園\"\r\n") + ] + return (rows.flatMap { $0 }, rows) + } + + func testShiftJISFileOpensAsShiftJIS() async throws { + let (source, table) = try await open(try quotedShiftJISExport().bytes) + XCTAssertEqual(source.dialect.encoding, .shiftJIS) + XCTAssertNotNil(source.origin) + XCTAssertEqual(table.cells(row: 0).map(\.text), ["001", "\u{9AD9}橋", "たかはし", "ｿﾌﾄｳｪｱの保守"]) + XCTAssertEqual(table.cells(row: 1).map(\.text), ["002", "山\u{FA11}", "やまさき", "\u{2170}の注文"]) + } + + func testUntouchedShiftJISRowsKeepTheirQuotesAndIBMExtensionCodes() async throws { + let export = try quotedShiftJISExport() + let (source, original) = try await open(export.bytes) + XCTAssertEqual(try save(original, source: source), export.bytes) + var table = original + table.setCell(.text("鈴木"), row: 2, column: 1) + let expected = export.rows[0] + export.rows[1] + export.rows[2] + + (try shiftJIS("003,鈴木,たなか,ポイント交換\r\n")) + export.rows[4] + XCTAssertEqual(try save(table, source: source), expected) + } + + func testDeletingAndReorderingShiftJISRowsCopiesTheRestExactly() async throws { + let export = try quotedShiftJISExport() + let (source, original) = try await open(export.bytes) + var table = original + table.deleteRows(IndexSet(integer: 1)) + XCTAssertEqual(try save(table, source: source), export.rows[0] + export.rows[1] + export.rows[3] + export.rows[4]) + var reordered = original + reordered.replaceRowOrder(.explicit([4, 2, 3, 1])) + XCTAssertEqual( + try save(reordered, source: source), + export.rows[0] + export.rows[4] + export.rows[2] + export.rows[3] + export.rows[1] + ) + } + + func testAnEditNearTheEndOfALargeShiftJISFileLeavesEveryOtherByte() async throws { + let header = try shiftJIS("コード,名称,住所\r\n") + let line = try shiftJIS(",東京都港区芝公園４－２－８,ｻﾞｲｺｶﾝﾘｽﾙ ポイント\r\n") + var bytes = header + let rowCount = 40_000 + for row in 0.. CSVDialect { - var dialect = CSVDialect.detect(from: data) + static func resolveDialect(in text: Data, options: CSVImportOptions) -> CSVDialect { + var dialect = CSVDialect.detect(from: text) if let byte = options.delimiter.byte { dialect.delimiter = byte } dialect.quoteChar = options.quoteCharacter.byte - if let forced = options.encoding.stringEncoding { - dialect.encoding = forced - } + dialect.escapeChar = options.quoteCharacter.byte + dialect.encoding = .utf8 return dialect } diff --git a/Plugins/CSVImportPlugin/CSVImportPlugin.swift b/Plugins/CSVImportPlugin/CSVImportPlugin.swift index 639b4a50c1..f8761cfffe 100644 --- a/Plugins/CSVImportPlugin/CSVImportPlugin.swift +++ b/Plugins/CSVImportPlugin/CSVImportPlugin.swift @@ -7,6 +7,7 @@ import Combine import Foundation import SwiftUI import TableProPluginKit +import TableProTabularIO final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "CSV Import" @@ -45,12 +46,26 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi let startTime = Date() let url = source.fileURL() - let data: Data + let text: CSVImportText do { - data = try Data(contentsOf: url, options: .mappedIfSafe) + text = try CSVImportText.contents(of: url, encoding: settings.encoding, isCancelled: { progress.isCancelled }) + } catch is TabularCancellation { + throw PluginImportCancellationError() } catch { throw PluginImportError.importFailed(error.localizedDescription) } + defer { text.removeTemporaryFile() } + if let line = text.firstUndecodableLine { + throw PluginImportError.importFailed(String( + format: String( + localized: "Line %@ is not valid %@ text. Choose the file's encoding in the import options, then import again.", + bundle: .main + ), + line.formatted(), + text.encoding.displayName + )) + } + let data = text.data let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) let parser = CSVStreamingParser(dialect: dialect) @@ -145,13 +160,7 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { - let data = try readDetectionPrefix(of: url) - return CSVImportParsing.detectFields(in: data, options: settings) - } - - private func readDetectionPrefix(of url: URL) throws -> Data { - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } - return handle.readData(ofLength: Self.detectionPrefixBytes) + let text = try CSVImportText.prefix(of: url, length: Self.detectionPrefixBytes, encoding: settings.encoding) + return CSVImportParsing.detectFields(in: text.data, options: settings) } } diff --git a/Plugins/CSVImportPlugin/CSVImportText.swift b/Plugins/CSVImportPlugin/CSVImportText.swift new file mode 100644 index 0000000000..4465602a03 --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportText.swift @@ -0,0 +1,87 @@ +// +// CSVImportText.swift +// CSVImportPlugin +// + +import Foundation +import TableProTabularIO + +struct CSVImportText { + let data: Data + let encoding: TabularTextEncoding + let firstUndecodableLine: Int? + private let temporaryURL: URL? + + static func prefix(of url: URL, length: Int, encoding option: CSVImportOptions.TextEncoding) throws -> CSVImportText { + let file = try Data(contentsOf: url, options: .alwaysMapped) + let head = file.prefix(length) + let sniff = resolvedEncoding(of: head, isWholeFile: head.count == file.count, option: option) + let transcoded = try TabularTextTranscoder.utf8Data( + from: file, + encoding: sniff.encoding, + skippingPrefix: sniff.byteOrderMarkLength, + wholeLinesWithin: length + ) + return CSVImportText( + data: transcoded.data, + encoding: sniff.encoding, + firstUndecodableLine: transcoded.firstUndecodableLine, + temporaryURL: nil + ) + } + + static func contents( + of url: URL, + encoding option: CSVImportOptions.TextEncoding, + copyingInto directory: URL = FileManager.default.temporaryDirectory, + isCancelled: () -> Bool = { false } + ) throws -> CSVImportText { + let file = try Data(contentsOf: url, options: .mappedIfSafe) + let sniff = resolvedEncoding(of: file, isWholeFile: true, option: option) + guard sniff.encoding != .utf8 else { + return CSVImportText( + data: file, + encoding: .utf8, + firstUndecodableLine: TabularTextTranscoder.firstInvalidUTF8Line( + in: file, + skippingPrefix: sniff.byteOrderMarkLength + ), + temporaryURL: nil + ) + } + let temporaryURL = directory.appendingPathComponent("TablePro-CSVImport-\(UUID().uuidString).csv") + do { + let transcoded = try TabularTextTranscoder.transcode( + file, + from: sniff.encoding, + skippingPrefix: sniff.byteOrderMarkLength, + to: temporaryURL, + isCancelled: isCancelled + ) + return CSVImportText( + data: try Data(contentsOf: temporaryURL, options: .alwaysMapped), + encoding: sniff.encoding, + firstUndecodableLine: transcoded.firstUndecodableLine, + temporaryURL: temporaryURL + ) + } catch { + try? FileManager.default.removeItem(at: temporaryURL) + throw error + } + } + + func removeTemporaryFile() { + guard let temporaryURL else { return } + try? FileManager.default.removeItem(at: temporaryURL) + } + + private static func resolvedEncoding( + of file: Data, + isWholeFile: Bool, + option: CSVImportOptions.TextEncoding + ) -> TabularEncodingSniff { + let sniff = TabularEncodingDetector.sniff(file, isWholeFile: isWholeFile) + guard let chosen = option.tabularEncoding, chosen != sniff.encoding else { return sniff } + return TabularEncodingSniff(encoding: chosen, byteOrderMarkLength: 0) + } +} diff --git a/Plugins/JSONImportPlugin/JSONImportParsing.swift b/Plugins/JSONImportPlugin/JSONImportParsing.swift index 59a4d3c8e1..a27414eec9 100644 --- a/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -94,11 +94,24 @@ enum JSONImportParsing { // MARK: - Source introspection + static let sampleLength = 256 * 1_024 + + private static let longestUTF8Tail = 3 + + static func utf8Text(ofSample sample: Data) -> String { + for dropped in 0...min(longestUTF8Tail, sample.count) { + if let text = String(data: sample.dropLast(dropped), encoding: .utf8) { + return text + } + } + return "" + } + static func sampleRawRows(at url: URL, targetTable: String?, limit: Int) throws -> [[String: Any]] { if isLineDelimited(url) { let handle = try FileHandle(forReadingFrom: url) defer { try? handle.close() } - let text = String(bytes: handle.readData(ofLength: 256 * 1_024), encoding: .utf8) ?? "" + let text = utf8Text(ofSample: handle.readData(ofLength: sampleLength)) var rows: [[String: Any]] = [] for line in text.split(separator: "\n") where rows.count < limit { let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/TablePro/Core/DataFiles/DataFileController+Saving.swift b/TablePro/Core/DataFiles/DataFileController+Saving.swift index bb8c1fdf9d..fbaeb0173d 100644 --- a/TablePro/Core/DataFiles/DataFileController+Saving.swift +++ b/TablePro/Core/DataFiles/DataFileController+Saving.swift @@ -10,7 +10,9 @@ import TableProTabularIO enum DataFileSaveError: LocalizedError { case notLoaded case unsupportedFormat - case unencodable(row: Int, column: Int, character: Character, encoding: String) + case unencodable(row: Int, column: String, character: String, encoding: String) + case unencodableInHiddenRow(column: String, character: String, encoding: String) + case unencodableColumnName(column: String, character: String, encoding: String) case invalidJSONValue(row: Int, column: String) case duplicateJSONKey(String) case failed(String) @@ -23,10 +25,24 @@ enum DataFileSaveError: LocalizedError { return String(localized: "This file cannot be saved in that format.") case .unencodable(let row, let column, let character, let encoding): return String( - format: String(localized: "Row %@, column %@ contains “%@”, which %@ cannot store. Choose Save As and pick UTF-8."), + format: String(localized: "Row %@, column “%@” contains %@, which %@ cannot store. Choose Save As and pick UTF-8."), row.formatted(), - column.formatted(), - String(character), + column, + character, + encoding + ) + case .unencodableInHiddenRow(let column, let character, let encoding): + return String( + format: String(localized: "A row the filter hides holds %@ in column “%@”, which %@ cannot store. Clear the filter to find it, or choose Save As and pick UTF-8."), + character, + column, + encoding + ) + case .unencodableColumnName(let column, let character, let encoding): + return String( + format: String(localized: "The column name “%@” contains %@, which %@ cannot store. Rename the column, or choose Save As and pick UTF-8."), + column, + character, encoding ) case .invalidJSONValue(let row, let column): @@ -45,6 +61,7 @@ enum DataFileSaveError: LocalizedError { extension DataFileController { func write(to url: URL, typeName: String) throws { + lastWrittenDialect = nil guard let table, loadState == .loaded else { throw DataFileSaveError.notLoaded } guard let format = DataFileKind.format(forSaveType: typeName) else { throw DataFileSaveError.unsupportedFormat @@ -131,13 +148,6 @@ extension DataFileController { } } - func adoptSaveEncoding() { - guard let saveEncoding, var updated = dialect, updated.encoding != saveEncoding else { return } - updated.encoding = saveEncoding - updated.hasByteOrderMark = !saveEncoding.byteOrderMark.isEmpty && saveEncoding != .utf8 - setDialect(updated) - } - private func writeDelimited(_ table: TabularTable, to url: URL, typeName: String) throws { let source = content?.delimitedSource let usesSource = source != nil && kind?.format == .delimited @@ -151,17 +161,48 @@ extension DataFileController { rows: table.outputRows(sourceHeaderNames: headerNames, copiesSourceRows: usesSource), endsWithLineTerminator: source?.index.endsWithLineTerminator ?? true ) + lastWrittenDialect = writer.dialect } catch TabularWriteError.unencodable(let row, let column, let character, let encoding) { - throw DataFileSaveError.unencodable( - row: row + 1, - column: column + 1, - character: character, - encoding: DataFileEncodingNames.name(for: encoding) - ) + throw unencodableError(outputRow: row, column: column, character: character, encoding: encoding, in: table) } catch TabularWriteError.couldNotCreate(let path) { throw DataFileSaveError.failed(String(format: String(localized: "Could not create %@."), path)) } catch TabularWriteError.writeFailed(let message) { throw DataFileSaveError.failed(message) } } + + private func unencodableError( + outputRow: Int, + column: Int, + character: Character, + encoding: TabularTextEncoding, + in table: TabularTable + ) -> DataFileSaveError { + let columnName = table.columns.indices.contains(column) ? table.columns[column].name : (column + 1).formatted() + let described = Self.describe(character) + let writesHeader = table.headerRowKey != nil + if writesHeader, outputRow == 0 { + return .unencodableColumnName(column: columnName, character: described, encoding: encoding.displayName) + } + let logicalRow = outputRow - (writesHeader ? 1 : 0) + guard logicalRow >= 0, logicalRow < table.rowCount else { + return .unencodableInHiddenRow(column: columnName, character: described, encoding: encoding.displayName) + } + let key = table.key(atRow: logicalRow) + guard let position = displayPosition(ofKey: key) else { + return .unencodableInHiddenRow(column: columnName, character: described, encoding: encoding.displayName) + } + reveal(key: key) + return .unencodable(row: position + 1, column: columnName, character: described, encoding: encoding.displayName) + } + + nonisolated static func describe(_ character: Character) -> String { + let isInvisible = character.unicodeScalars.allSatisfy { scalar in + scalar.properties.isWhitespace || [.control, .format].contains(scalar.properties.generalCategory) + } + guard isInvisible else { return "“\(character)”" } + return character.unicodeScalars + .map { String(format: "U+%04X %@", $0.value, $0.properties.name ?? "") } + .joined(separator: ", ") + } } diff --git a/TablePro/Core/DataFiles/DataFileController.swift b/TablePro/Core/DataFiles/DataFileController.swift index 22f621db17..dfb8cddf0b 100644 --- a/TablePro/Core/DataFiles/DataFileController.swift +++ b/TablePro/Core/DataFiles/DataFileController.swift @@ -40,7 +40,10 @@ final class DataFileController: ObservableObject { @Published private(set) var selectedSheetIndex = 0 @Published private(set) var dialect: DelimitedDialect? var saveEncoding: TabularTextEncoding? + var lastWrittenDialect: DelimitedDialect? @Published private(set) var raggedRowCount = 0 + @Published private(set) var undecodableLineCount = 0 + private(set) var chosenDialect: DelimitedDialect? @Published var filterState = TabFilterState() @Published var searchText = "" @@ -119,9 +122,12 @@ final class DataFileController: ObservableObject { queryTask?.cancel() mutationTask?.cancel() self.kind = kind + if let dialectOverride { + chosenDialect = dialectOverride + } loadState = .loading loadProgress = 0 - let request = DataFileLoadRequest(url: url, kind: kind, dialectOverride: dialectOverride) + let request = DataFileLoadRequest(url: url, kind: kind, dialectOverride: chosenDialect) loadTask = Task { [weak self] in await self?.performLoad(request) } @@ -163,6 +169,7 @@ final class DataFileController: ObservableObject { sheets = content.sheets dialect = content.dialect raggedRowCount = content.raggedRowCount + undecodableLineCount = content.undecodableLineCount selectedSheetIndex = content.initialSheetIndex undoManager?.removeAllActions() installSheet(at: content.initialSheetIndex) @@ -481,5 +488,15 @@ final class DataFileController: ObservableObject { func setDialect(_ newDialect: DelimitedDialect) { dialect = newDialect + guard chosenDialect != nil else { return } + chosenDialect = newDialect + } + + func adoptWrittenDialect() { + guard let written = lastWrittenDialect else { return } + if written.encoding != dialect?.encoding { + undecodableLineCount = 0 + } + setDialect(written) } } diff --git a/TablePro/Core/DataFiles/DataFileCountPhrase.swift b/TablePro/Core/DataFiles/DataFileCountPhrase.swift index 834976622d..8ebebf27d6 100644 --- a/TablePro/Core/DataFiles/DataFileCountPhrase.swift +++ b/TablePro/Core/DataFiles/DataFileCountPhrase.swift @@ -35,4 +35,10 @@ enum DataFileCountPhrase { ? String(localized: "1 row has a different number of fields") : String(format: String(localized: "%@ rows have a different number of fields"), count.formatted()) } + + static func undecodableLines(_ count: Int) -> String { + count == 1 + ? String(localized: "1 line has unreadable characters") + : String(format: String(localized: "%@ lines have unreadable characters"), count.formatted()) + } } diff --git a/TablePro/Core/DataFiles/DataFileDocument.swift b/TablePro/Core/DataFiles/DataFileDocument.swift index e222674832..68021ebc1b 100644 --- a/TablePro/Core/DataFiles/DataFileDocument.swift +++ b/TablePro/Core/DataFiles/DataFileDocument.swift @@ -85,11 +85,11 @@ final class DataFileDocument: NSDocument { guard controller.offersSaveEncoding else { return super.prepareSavePanel(savePanel) } let popUp = NSPopUpButton(frame: .zero, pullsDown: false) for encoding in TabularTextEncoding.allCases { - popUp.addItem(withTitle: DataFileEncodingNames.name(for: encoding)) + popUp.addItem(withTitle: encoding.displayName) popUp.lastItem?.representedObject = encoding.rawValue } let current = controller.dialect?.encoding ?? .utf8 - popUp.selectItem(withTitle: DataFileEncodingNames.name(for: current)) + popUp.selectItem(withTitle: current.displayName) popUp.setAccessibilityLabel(String(localized: "Text Encoding")) let row = NSStackView(views: [NSTextField(labelWithString: String(localized: "Text Encoding:")), popUp]) row.orientation = .horizontal @@ -115,7 +115,7 @@ final class DataFileDocument: NSDocument { controller.saveEncoding = saveOperation == .saveOperation ? nil : chosen super.save(to: url, ofType: typeName, for: saveOperation) { [weak self] error in if error == nil, saveOperation == .saveAsOperation { - self?.controller.adoptSaveEncoding() + self?.controller.adoptWrittenDialect() } self?.controller.saveEncoding = nil completionHandler(error) diff --git a/TablePro/Core/DataFiles/DataFileLoader.swift b/TablePro/Core/DataFiles/DataFileLoader.swift index 1d69b80456..a6ad7b0f0a 100644 --- a/TablePro/Core/DataFiles/DataFileLoader.swift +++ b/TablePro/Core/DataFiles/DataFileLoader.swift @@ -30,6 +30,7 @@ struct DataFileContent: Sendable { let workbook: XLSXWorkbook? let dialect: DelimitedDialect? let raggedRowCount: Int + let undecodableLineCount: Int } struct DataFileLoadedSheet: Sendable { @@ -45,7 +46,6 @@ extension Error { enum DataFileLoadError: LocalizedError, Equatable { case unreadable(String) - case undecodable(String) case unsupported(String) case invalidJSON(JSONTableError) @@ -53,8 +53,6 @@ enum DataFileLoadError: LocalizedError, Equatable { switch self { case .unreadable(let message): return String(format: String(localized: "The file could not be read: %@"), message) - case .undecodable(let encoding): - return String(format: String(localized: "The file is not valid %@ text."), encoding) case .unsupported(let message): return message case .invalidJSON(let error): @@ -146,7 +144,8 @@ enum DataFileLoader { jsonSource: source, workbook: nil, dialect: nil, - raggedRowCount: 0 + raggedRowCount: 0, + undecodableLineCount: 0 ) } @@ -196,7 +195,8 @@ enum DataFileLoader { jsonSource: nil, workbook: workbook, dialect: nil, - raggedRowCount: 0 + raggedRowCount: 0, + undecodableLineCount: 0 ) } @@ -215,50 +215,25 @@ enum DataFileLoader { progress: @escaping @Sendable (Double) -> Void ) async throws -> DataFileContent { let original = try mappedData(at: snapshotURL) - let sniff = original.withUnsafeBytes { raw in - DelimitedDialectDetector.sniffEncoding(raw.bindMemory(to: UInt8.self)) - } - let fileEncoding = request.dialectOverride?.encoding ?? sniff.encoding - let prefixLength = fileEncoding == sniff.encoding ? sniff.byteOrderMarkLength : 0 - let bytes: Data - let byteEncoding: TabularTextEncoding - let contentStart: Int - if fileEncoding.readsInPlace { - bytes = original - byteEncoding = fileEncoding - contentStart = prefixLength - } else { - do { - let converted = try TabularTextTranscoder.utf8Data(from: original, encoding: fileEncoding, skippingPrefix: prefixLength) - let convertedURL = workingCopy.file(named: "utf8-\(UUID().uuidString).txt") - try converted.write(to: convertedURL) - bytes = try mappedData(at: convertedURL) - } catch is TabularTranscodingError { - throw DataFileLoadError.undecodable(DataFileEncodingNames.name(for: fileEncoding)) - } - byteEncoding = .utf8 - contentStart = 0 - } - try Task.checkCancellation() - let detected = bytes.withUnsafeBytes { raw in - DelimitedDialectDetector.detect( - raw.bindMemory(to: UInt8.self), - contentStart: contentStart, - encoding: fileEncoding, - hasByteOrderMark: sniff.hasByteOrderMark && fileEncoding == sniff.encoding, - fileExtension: request.kind.contentExtension + let contents: DelimitedFileContents + do { + contents = try await DelimitedFileReader.read( + original, + fileExtension: request.kind.contentExtension, + dialectOverride: request.dialectOverride, + transcodedFileURL: workingCopy.file(named: "utf8-\(UUID().uuidString).txt"), + progress: progress, + isCancelled: { Task.isCancelled } ) + } catch TabularWriteError.couldNotCreate(let path) { + throw DataFileLoadError.unreadable(String(format: String(localized: "Could not create %@."), path)) + } catch TabularWriteError.writeFailed(let message) { + throw DataFileLoadError.unreadable(message) + } catch let error as CocoaError { + throw DataFileLoadError.unreadable(error.localizedDescription) } - let dialect = request.dialectOverride ?? detected - let source = try await DelimitedSourceBuilder.build( - bytes: bytes, - dialect: dialect, - byteEncoding: byteEncoding, - contentStart: contentStart, - progress: { progress($0 * 0.95) }, - isCancelled: { Task.isCancelled } - ) - let table = TabularTable(source: source, usesFirstRowAsHeader: dialect.hasHeaderRow) + let source = contents.source + let table = TabularTable(source: source, usesFirstRowAsHeader: source.dialect.hasHeaderRow) let kinds = TabularTypeInference.inferKinds(of: table) progress(1) let sheet = DataFileSheet( @@ -275,24 +250,9 @@ enum DataFileLoader { delimitedSource: source, jsonSource: nil, workbook: nil, - dialect: dialect, - raggedRowCount: source.raggedRowCount + dialect: source.dialect, + raggedRowCount: source.raggedRowCount, + undecodableLineCount: contents.undecodableLineCount ) } } - -enum DataFileEncodingNames { - static func name(for encoding: TabularTextEncoding) -> String { - switch encoding { - case .utf8: return "UTF-8" - case .utf16LittleEndian: return "UTF-16 LE" - case .utf16BigEndian: return "UTF-16 BE" - case .windows1252: return "Windows-1252" - case .isoLatin1: return "ISO Latin 1" - case .shiftJIS: return "Shift JIS" - case .gb18030: return "GB 18030" - case .big5: return "Big5" - case .eucKR: return "EUC-KR" - } - } -} diff --git a/TablePro/Core/DataFiles/DataFileWorkingCopy.swift b/TablePro/Core/DataFiles/DataFileWorkingCopy.swift index 9e3e855a5a..17e547c8db 100644 --- a/TablePro/Core/DataFiles/DataFileWorkingCopy.swift +++ b/TablePro/Core/DataFiles/DataFileWorkingCopy.swift @@ -38,10 +38,6 @@ final class DataFileWorkingCopy: @unchecked Sendable { try await GzipProcess.decompress(source: url, destination: destination) return destination } - let values = try? url.resourceValues(forKeys: [.volumeSupportsFileCloningKey, .volumeIsLocalKey]) - let supportsCloning = values?.volumeSupportsFileCloning ?? false - let isLocal = values?.volumeIsLocal ?? false - guard supportsCloning || !isLocal else { return url } let destination = file(named: "content-\(UUID().uuidString).\(kind.contentExtension)") try FileManager.default.copyItem(at: url, to: destination) return destination diff --git a/TablePro/Core/Plugins/SqlFileImportSource.swift b/TablePro/Core/Plugins/SqlFileImportSource.swift index 5b30677364..560b37de92 100644 --- a/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -14,6 +14,7 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { private let url: URL private let encoding: String.Encoding private let grammar: SQLLexicalGrammar + private let family: TransactionEngineFamily private let parser = SQLFileParser() private let externalDecompressedURL: URL? @@ -24,12 +25,14 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar = .ansi, + family: TransactionEngineFamily = .other, decompressedURL: URL? = nil, ownsDecompressedFile: Bool? = nil ) { self.url = url self.encoding = encoding self.grammar = grammar + self.family = family self.externalDecompressedURL = decompressedURL self.ownsDecompressedFile = ownsDecompressedFile ?? (decompressedURL == nil) } @@ -51,7 +54,25 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { let fileURL = try await resolveURL() - return parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar) + let parsed = parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar) + guard !sendsFileBytesUnchanged else { return parsed } + return Self.sentAsUTF8(parsed, decodedFrom: encoding, family: family, grammar: grammar) + } + + private var sendsFileBytesUnchanged: Bool { + encoding == .utf8 || encoding == .ascii + } + + private static func sentAsUTF8( + _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, + decodedFrom encoding: String.Encoding, + family: TransactionEngineFamily, + grammar: SQLLexicalGrammar + ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + let rewriting = TranscodedDumpStatements(statements, decodedFrom: encoding, family: family, grammar: grammar) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) } /// `ownsDecompressedFile` answers only whether the caller handed over a file to delete. A file @@ -113,3 +134,71 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { return result } } + +internal enum SqlFileImportError: LocalizedError, Equatable { + case unrecoverableBinaryLiteral(line: Int, encoding: String) + + var errorDescription: String? { + switch self { + case .unrecoverableBinaryLiteral(let line, let encoding): + return String( + format: String(localized: "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again."), + line.formatted(), + encoding + ) + } + } +} + +private final class TranscodedDumpStatements: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") + + private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator + private let encoding: String.Encoding + private let family: TransactionEngineFamily + private let grammar: SQLLexicalGrammar + private var backslashEscapesAreOn = true + + init( + _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, + decodedFrom encoding: String.Encoding, + family: TransactionEngineFamily, + grammar: SQLLexicalGrammar + ) { + self.statements = statements.makeAsyncIterator() + self.encoding = encoding + self.family = family + self.grammar = grammar + } + + func next() async throws -> (statement: String, lineNumber: Int)? { + guard let (statement, lineNumber) = try await statements.next() else { return nil } + let literals = try rewrittenLiterals(of: statement, at: lineNumber) ?? statement + guard let rewritten = SQLClientEncodingRewriter.rewritten(literals, family: family, grammar: grammar) else { + return (literals, lineNumber) + } + Self.logger.info("Kept the session's client encoding at line \(lineNumber, privacy: .public), since the import sends UTF-8") + return (rewritten, lineNumber) + } + + private func rewrittenLiterals(of statement: String, at lineNumber: Int) throws -> String? { + guard family == .mysql else { return nil } + defer { + backslashEscapesAreOn = SQLTranscodedLiteralRewriter.backslashEscapesAreOn( + after: statement, + currently: backslashEscapesAreOn, + grammar: grammar + ) + } + do { + return try SQLTranscodedLiteralRewriter.rewritten( + statement, + decodedFrom: encoding, + backslashEscapesAreOn: backslashEscapesAreOn, + grammar: grammar + ) + } catch SQLTranscodedLiteralError.unrecoverableBinaryLiteral { + throw SqlFileImportError.unrecoverableBinaryLiteral(line: lineNumber, encoding: String.localizedName(of: encoding)) + } + } +} diff --git a/TablePro/Core/Services/Export/ImportService.swift b/TablePro/Core/Services/Export/ImportService.swift index f79877b44f..bf90442a5a 100644 --- a/TablePro/Core/Services/Export/ImportService.swift +++ b/TablePro/Core/Services/Export/ImportService.swift @@ -79,6 +79,7 @@ final class ImportService: ObservableObject { url: url, encoding: encoding, grammar: connection.type.lexicalGrammar, + family: TransactionEngineFamily.of(connection.type), decompressedURL: decompressedURL, ownsDecompressedFile: ownsDecompressedFile ) diff --git a/TablePro/Core/Utilities/File/FileTextEncoding.swift b/TablePro/Core/Utilities/File/FileTextEncoding.swift index c18c5abadb..1eebcf71d1 100644 --- a/TablePro/Core/Utilities/File/FileTextEncoding.swift +++ b/TablePro/Core/Utilities/File/FileTextEncoding.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProTabularIO internal struct FileTextEncoding: Hashable, Sendable { let encoding: String.Encoding @@ -12,6 +13,8 @@ internal struct FileTextEncoding: Hashable, Sendable { static let utf8 = FileTextEncoding(encoding: .utf8) + private static let japaneseEncodings: Set = [.shiftJIS, .japaneseEUC] + init( encoding: String.Encoding, byteOrderMark: ByteOrderMark? = nil, @@ -30,7 +33,8 @@ internal struct FileTextEncoding: Hashable, Sendable { func bytes(of text: String) -> Data? { let byteOrderedEncoding = byteOrderMark?.byteOrderedEncoding ?? encoding.unmarkedByteOrder - guard let body = text.data(using: byteOrderedEncoding, allowLossyConversion: false) else { return nil } + let storable = Self.japaneseEncodings.contains(encoding) ? TabularJISRoman.folded(text) : text + guard let body = storable.data(using: byteOrderedEncoding, allowLossyConversion: false) else { return nil } guard let byteOrderMark else { return body } return Data(byteOrderMark.bytes) + body } diff --git a/TablePro/Core/Utilities/File/FileTextLoader.swift b/TablePro/Core/Utilities/File/FileTextLoader.swift index d89de3728d..50f21d8803 100644 --- a/TablePro/Core/Utilities/File/FileTextLoader.swift +++ b/TablePro/Core/Utilities/File/FileTextLoader.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProTabularIO internal enum FileTextLoader { struct LoadedText: Sendable { @@ -31,18 +32,52 @@ internal enum FileTextLoader { return try readByteOrderMarked(url, stamp: stamp) } let attribute = TextEncodingAttribute.read(from: url) - var detected: String.Encoding = .utf8 - if let content = try? String(contentsOf: url, usedEncoding: &detected) { - let textEncoding = FileTextEncoding(encoding: detected, attributeOnDisk: attribute) - return LoadedText(content: content, textEncoding: textEncoding, stamp: stamp) + if attribute != nil { + var recorded: String.Encoding = .utf8 + if let content = try? String(contentsOf: url, usedEncoding: &recorded) { + let textEncoding = FileTextEncoding(encoding: recorded, attributeOnDisk: attribute) + return LoadedText(content: content, textEncoding: textEncoding, stamp: stamp) + } } - if let content = try? String(contentsOf: url, encoding: .utf8) { + let data = try Data(contentsOf: url) + let detected = multiByteEncoding(of: data, isWholeFile: true) + if let detected, detected.codeUnitLength > 1, let content = String(data: data, encoding: detected) { + return LoadedText(content: content, textEncoding: FileTextEncoding(encoding: detected), stamp: stamp) + } + if let content = String(data: data, encoding: .utf8) { return LoadedText(content: content, textEncoding: .utf8, stamp: stamp) } - let content = try String(contentsOf: url, encoding: .isoLatin1) + if let detected, let content = String(data: data, encoding: detected) { + return LoadedText(content: content, textEncoding: FileTextEncoding(encoding: detected), stamp: stamp) + } + guard let content = String(data: data, encoding: .isoLatin1) else { + throw CocoaError(.fileReadCorruptFile, userInfo: [NSFilePathErrorKey: url.path]) + } return LoadedText(content: content, textEncoding: FileTextEncoding(encoding: .isoLatin1), stamp: stamp) } + static func resolvedEncoding(of url: URL) -> String.Encoding? { + load(url)?.encoding + } + + static func load(_ url: URL, as textEncoding: FileTextEncoding) -> LoadedText? { + let stamp = FileStamp.read(url) + guard let data = try? Data(contentsOf: url) else { return nil } + guard textEncoding.byteOrderMark.map({ data.starts(with: $0.bytes) }) ?? (ByteOrderMark.leading(data) == nil) else { + return nil + } + let body = textEncoding.byteOrderMark.map { data.dropFirst($0.bytes.count) } ?? data[...] + let encoding = textEncoding.byteOrderMark?.byteOrderedEncoding ?? textEncoding.encoding + guard let content = String(data: Data(body), encoding: encoding) else { return nil } + return LoadedText(content: content, textEncoding: textEncoding, stamp: stamp) + } + + private static func multiByteEncoding(of data: Data, isWholeFile: Bool) -> String.Encoding? { + let detected = TabularEncodingDetector.sniff(data, isWholeFile: isWholeFile).encoding + guard !detected.readsInPlace else { return nil } + return detected.foundationEncoding + } + static func decode(_ data: Data, declaredEncoding: String.Encoding? = nil) -> String? { if declaredEncoding != nil, ByteOrderMark.leading(data) == nil, let utf8 = String(data: data, encoding: .utf8) { return utf8 @@ -56,12 +91,14 @@ internal enum FileTextLoader { let attribute = TextEncodingAttribute.read(from: url) guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } - guard let bytes = try? handle.read(upToCount: maxBytes + TextPrefixDecoder.lookaheadLength), - let decoded = TextPrefixDecoder.decode( - bytes, - prefixLength: maxBytes, - declaredEncoding: attribute?.encoding - ) else { return nil } + guard let bytes = try? handle.read(upToCount: maxBytes + TextPrefixDecoder.lookaheadLength) else { return nil } + let isWholeFile = bytes.count < maxBytes + TextPrefixDecoder.lookaheadLength + let declaredEncoding = attribute?.encoding ?? multiByteEncoding(of: bytes, isWholeFile: isWholeFile) + guard let decoded = TextPrefixDecoder.decode( + bytes, + prefixLength: maxBytes, + declaredEncoding: declaredEncoding + ) else { return nil } return LoadedText( content: decoded.content, textEncoding: textEncoding(of: decoded, attribute: attribute), diff --git a/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift b/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift new file mode 100644 index 0000000000..c2e9876cee --- /dev/null +++ b/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift @@ -0,0 +1,199 @@ +// +// SQLClientEncodingRewriter.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +internal enum SQLClientEncodingRewriter { + private struct Replacement { + let range: NSRange + let text: String + } + + private static let mysqlSessionCharacterSet = "@@session.character_set_client" + private static let mysqlKeepSessionCharacterSet = "character_set_client = @@session.character_set_client" + private static let postgresEncoding = "'UTF8'" + private static let unicodeSpellings: Set = ["UTF8", "UTF8MB4", "UTF8MB3", "UNICODE"] + private static let mysqlSessionScopes: Set = ["SESSION", "LOCAL"] + private static let mysqlServerScopes: Set = ["GLOBAL", "PERSIST", "PERSIST_ONLY"] + private static let postgresScopes: Set = ["SESSION", "LOCAL"] + private static let clientCharacterSetVariable = "CHARACTER_SET_CLIENT" + + internal static func rewritten( + _ statement: String, + family: TransactionEngineFamily, + grammar: SQLLexicalGrammar + ) -> String? { + let text = statement as NSString + var cursor = SQLTokenCursor(text, grammar: grammar) + guard cursor.next()?.word == "SET" else { return nil } + let replacements: [Replacement] + switch family { + case .mysql: + replacements = mysqlClientCharacterSets(&cursor, in: text) + case .postgres, .redshift, .cockroach: + replacements = postgresClientEncoding(&cursor, in: text).map { [Replacement(range: $0, text: postgresEncoding)] } ?? [] + case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .other: + return nil + } + guard !replacements.isEmpty else { return nil } + let result = NSMutableString(string: text) + for replacement in replacements.reversed() { + result.replaceCharacters(in: replacement.range, with: replacement.text) + } + return result as String + } + + private static func mysqlClientCharacterSets(_ cursor: inout SQLTokenCursor, in text: NSString) -> [Replacement] { + var replacements: [Replacement] = [] + var listIsServerScoped = false + var head = cursor.next() + while let token = head { + if let replacement = mysqlDeclaration( + startingAt: token, + cursor: &cursor, + in: text, + listIsServerScoped: &listIsServerScoped + ) { + replacements.append(replacement) + } + head = nextElementHead(&cursor) + } + return replacements + } + + private static func mysqlDeclaration( + startingAt token: SQLTokenCursor.Token, + cursor: inout SQLTokenCursor, + in text: NSString, + listIsServerScoped: inout Bool + ) -> Replacement? { + var current = token + while let word = current.word, mysqlSessionScopes.contains(word) || mysqlServerScopes.contains(word) { + listIsServerScoped = mysqlServerScopes.contains(word) + guard let next = cursor.next() else { return nil } + current = next + } + let elementStart = cursor.tokenRange.location + switch current.word { + case "NAMES": + return characterSetValue(&cursor, in: text, allowsCollation: true).map { + Replacement(range: range(from: elementStart, through: $0), text: mysqlKeepSessionCharacterSet) + } + case "CHARSET": + return characterSetValue(&cursor, in: text, allowsCollation: false).map { + Replacement(range: range(from: elementStart, through: $0), text: mysqlKeepSessionCharacterSet) + } + case "CHARACTER": + guard cursor.next()?.word == "SET" else { return nil } + return characterSetValue(&cursor, in: text, allowsCollation: false).map { + Replacement(range: range(from: elementStart, through: $0), text: mysqlKeepSessionCharacterSet) + } + default: + guard let isSessionVariable = clientCharacterSetVariableScope(current, cursor: &cursor), + isSessionVariable || !listIsServerScoped, + cursor.next()?.isSymbol(SQLTokenCursor.equals) == true else { + return nil + } + return characterSetValue(&cursor, in: text, allowsCollation: false).map { + Replacement(range: $0, text: mysqlSessionCharacterSet) + } + } + } + + private static func clientCharacterSetVariableScope( + _ token: SQLTokenCursor.Token, + cursor: inout SQLTokenCursor + ) -> Bool? { + guard let name = token.identifier else { return nil } + switch name { + case clientCharacterSetVariable: + return false + case "@@" + clientCharacterSetVariable: + return true + case "@@SESSION", "@@LOCAL": + guard cursor.next()?.isSymbol(SQLTokenCursor.period) == true, + cursor.next()?.identifier == clientCharacterSetVariable else { + return nil + } + return true + default: + return nil + } + } + + private static func postgresClientEncoding(_ cursor: inout SQLTokenCursor, in text: NSString) -> NSRange? { + guard var token = cursor.next() else { return nil } + if let word = token.word, postgresScopes.contains(word) { + guard let next = cursor.next() else { return nil } + token = next + } + switch token.identifier { + case "CLIENT_ENCODING": + guard let separator = cursor.next(), + separator.isSymbol(SQLTokenCursor.equals) || separator.word == "TO" else { + return nil + } + return characterSetValue(&cursor, in: text, allowsCollation: false) + case "NAMES": + return characterSetValue(&cursor, in: text, allowsCollation: false) + default: + return nil + } + } + + private static func characterSetValue( + _ cursor: inout SQLTokenCursor, + in text: NSString, + allowsCollation: Bool + ) -> NSRange? { + guard let value = cursor.next() else { return nil } + let valueRange = cursor.tokenRange + guard let name = characterSetName(of: value, range: valueRange, in: text), + !unicodeSpellings.contains(normalized(name)) else { + return nil + } + guard allowsCollation else { return valueRange } + var lookahead = cursor + guard lookahead.next()?.word == "COLLATE", lookahead.next() != nil else { return valueRange } + cursor = lookahead + return NSRange(location: valueRange.location, length: cursor.location - valueRange.location) + } + + private static func range(from start: Int, through end: NSRange) -> NSRange { + NSRange(location: start, length: NSMaxRange(end) - start) + } + + private static func characterSetName(of token: SQLTokenCursor.Token, range: NSRange, in text: NSString) -> String? { + switch token { + case .word(let word): + guard word != "DEFAULT", !word.hasPrefix("@"), let first = word.unicodeScalars.first, + !CharacterSet.decimalDigits.contains(first) else { + return nil + } + return word + case .quotedIdentifier(let name): + return name + case .literal: + guard range.length >= 2 else { return nil } + return text.substring(with: NSRange(location: range.location + 1, length: range.length - 2)) + case .symbol: + return nil + } + } + + private static func normalized(_ name: String) -> String { + String(name.uppercased().unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) }) + } + + private static func nextElementHead(_ cursor: inout SQLTokenCursor) -> SQLTokenCursor.Token? { + while let token = cursor.next() { + if token.isSymbol(SQLTokenCursor.comma), cursor.parenDepth == 0 { + return cursor.next() + } + } + return nil + } +} diff --git a/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift b/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift new file mode 100644 index 0000000000..311ba4b760 --- /dev/null +++ b/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift @@ -0,0 +1,77 @@ +// +// SQLDumpEncodingChoice.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar +import TableProTabularIO + +internal enum SQLDumpEncodingChoice { + private static let declarationScanLength = 65_536 + private static let declarationPattern = try? NSRegularExpression(pattern: [ + #"(?i)\A(?:\s+|--[^\n]*(?:\n|\z)|#[^\n]*(?:\n|\z)|/\*(?!M?!)[\s\S]*?\*/)*"#, + #"(?:/\*M?!\d*\s*)?SET\s+(?:(?:SESSION|LOCAL)\s+)?"#, + #"(?:NAMES|CHARACTER\s+SET|CHARSET|character_set_client\s*=|client_encoding\s*(?:=|TO))"#, + #"\s*['"]?([A-Za-z0-9_-]+)"# + ].joined()) + + private static let declaredNames: [String: ImportEncoding] = [ + "UTF8": .utf8, "UTF8MB4": .utf8, "UTF8MB3": .utf8, "UNICODE": .utf8, + "CP1252": .windows1252, "WIN1252": .windows1252, "WINDOWS1252": .windows1252, + "LATIN1MYSQL": .windows1252, "LATIN1": .latin1, "ISO88591": .latin1, + "ASCII": .ascii, "SQLASCII": .ascii, + "CP932": .shiftJIS, "SJIS": .shiftJIS, "SHIFTJIS": .shiftJIS, "SHIFTJIS2004": .shiftJIS, + "UJIS": .eucJP, "EUCJPMS": .eucJP, "EUCJP": .eucJP, + "GBK": .gb18030, "GB2312": .gb18030, "GB18030": .gb18030, "EUCCN": .gb18030, + "BIG5": .big5, "CP950": .big5, + "EUCKR": .eucKR, "UHC": .eucKR, "CP949": .eucKR + ] + + internal static func encoding( + replacing selected: ImportEncoding, + forPreview preview: Data, + isWholeFile: Bool, + family: TransactionEngineFamily, + grammar: SQLLexicalGrammar + ) -> ImportEncoding? { + if let declared = declaredEncoding(in: preview, family: family, grammar: grammar) { + guard declared != selected, decodes(preview, as: declared) else { return nil } + return declared + } + let sniff = TabularEncodingDetector.sniff(preview, isWholeFile: isWholeFile) + guard let detected = ImportEncoding(detected: sniff.encoding), detected != selected, + !detected.canHideABackslashInsideACharacter, decodes(preview, as: detected) else { + return nil + } + return detected + } + + internal static func declaredEncoding( + in preview: Data, + family: TransactionEngineFamily, + grammar: SQLLexicalGrammar + ) -> ImportEncoding? { + guard let declarationPattern, + let head = String(data: preview.prefix(declarationScanLength), encoding: .isoLatin1) else { + return nil + } + for statement in SQLStatementScanner.allStatements(in: head, grammar: grammar) { + let text = statement as NSString + guard let match = declarationPattern.firstMatch(in: statement, range: NSRange(location: 0, length: text.length)) else { + continue + } + let name = text.substring(with: match.range(at: 1)).uppercased().filter { $0.isLetter || $0.isNumber } + let key = name == "LATIN1" && family == .mysql ? "LATIN1MYSQL" : name + if let encoding = declaredNames[key] { + return encoding + } + } + return nil + } + + private static func decodes(_ preview: Data, as encoding: ImportEncoding) -> Bool { + var decoder = SQLChunkDecoder(encoding: encoding.encoding) + return decoder.decode(preview) != nil + } +} diff --git a/TablePro/Core/Utilities/SQL/SQLTokenCursor.swift b/TablePro/Core/Utilities/SQL/SQLTokenCursor.swift index 9e1319b97b..6e5bb7c3c3 100644 --- a/TablePro/Core/Utilities/SQL/SQLTokenCursor.swift +++ b/TablePro/Core/Utilities/SQL/SQLTokenCursor.swift @@ -45,9 +45,14 @@ internal struct SQLTokenCursor { private var conditionalDepth = 0 internal private(set) var parenDepth = 0 + internal private(set) var tokenStart = 0 internal var location: Int { index } + internal var tokenRange: NSRange { + NSRange(location: tokenStart, length: index - tokenStart) + } + internal init(_ text: NSString, grammar: SQLLexicalGrammar) { self.text = text self.grammar = grammar @@ -68,6 +73,7 @@ internal struct SQLTokenCursor { } if skipsNonCode(character) { continue } if character == SqlLexer.semicolon, parenDepth == 0 { return nil } + tokenStart = index return token(startingWith: character) } return nil diff --git a/TablePro/Core/Utilities/SQL/SQLTranscodedLiteralRewriter.swift b/TablePro/Core/Utilities/SQL/SQLTranscodedLiteralRewriter.swift new file mode 100644 index 0000000000..d702390db3 --- /dev/null +++ b/TablePro/Core/Utilities/SQL/SQLTranscodedLiteralRewriter.swift @@ -0,0 +1,267 @@ +// +// SQLTranscodedLiteralRewriter.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +internal enum SQLTranscodedLiteralError: Error, Equatable { + case unrecoverableBinaryLiteral +} + +internal enum SQLTranscodedLiteralRewriter { + private struct Introducer { + let range: NSRange + let characterSet: String + } + + private struct Replacement { + let range: NSRange + let text: String + } + + private enum SQLModeAssignment { + case literal(String) + case unknown + } + + private struct LiteralContext { + let encoding: String.Encoding + let backslashEscapesAreOn: Bool + let grammar: SQLLexicalGrammar + } + + private static let underscore = UInt16(UnicodeScalar("_").value) + private static let unicodeCharacterSets: Set = ["utf8", "utf8mb4", "utf8mb3"] + private static let binaryCharacterSet = "binary" + private static let exactlyRecoverableEncodings: Set = [.isoLatin1, .windowsCP1252] + private static let escapedBytes: [UInt8: UInt8] = [ + UInt8(ascii: "0"): 0x00, UInt8(ascii: "b"): 0x08, UInt8(ascii: "n"): 0x0A, + UInt8(ascii: "r"): 0x0D, UInt8(ascii: "t"): 0x09, UInt8(ascii: "Z"): 0x1A + ] + private static let backslash = UInt8(ascii: "\\") + private static let noBackslashEscapes = "NO_BACKSLASH_ESCAPES" + private static let sqlModeVariable = "SQL_MODE" + private static let sessionScopes: Set = ["SESSION", "LOCAL"] + private static let serverScopes: Set = ["GLOBAL", "PERSIST", "PERSIST_ONLY"] + private static let percent = UInt8(ascii: "%") + + internal static func rewritten( + _ statement: String, + decodedFrom encoding: String.Encoding, + backslashEscapesAreOn: Bool = true, + grammar: SQLLexicalGrammar + ) throws -> String? { + let text = statement as NSString + guard text.range(of: "_").location != NSNotFound else { return nil } + let context = LiteralContext(encoding: encoding, backslashEscapesAreOn: backslashEscapesAreOn, grammar: grammar) + let replacements = try literalReplacements(in: text, context: context) + guard !replacements.isEmpty else { return nil } + let result = NSMutableString(string: text) + for replacement in replacements.reversed() { + result.replaceCharacters(in: replacement.range, with: replacement.text) + } + return result as String + } + + private static func literalReplacements(in text: NSString, context: LiteralContext) throws -> [Replacement] { + let grammar = context.grammar + let length = text.length + var replacements: [Replacement] = [] + var introducer: Introducer? + var index = 0 + while index < length { + let character = text.character(at: index) + if SqlLexer.isWhitespace(character) { + index += 1 + continue + } + if let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: length) { + introducer = nil + index += opener + continue + } + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + if span.kind == .quoted, isStringQuote(text.character(at: span.start)), let pending = introducer, + let replacement = try replacement(for: pending, literal: span, in: text, context: context) { + replacements.append(replacement) + } + if !span.kind.isComment { + introducer = nil + } + index = max(span.end, index + 1) + continue + } + guard SQLNonCodeSpan.isWordUnit(character) else { + introducer = nil + index += 1 + continue + } + let wordStart = index + while index < length, SQLNonCodeSpan.isWordUnit(text.character(at: index)) { + index += 1 + } + introducer = introducerWord(in: text, range: NSRange(location: wordStart, length: index - wordStart)) + } + return replacements + } + + internal static func backslashEscapesAreOn( + after statement: String, + currently isOn: Bool, + grammar: SQLLexicalGrammar + ) -> Bool { + let text = statement as NSString + guard text.range(of: "sql_mode", options: .caseInsensitive).location != NSNotFound else { return isOn } + var cursor = SQLTokenCursor(text, grammar: grammar) + guard cursor.next()?.word == "SET" else { return isOn } + var result = isOn + var listIsServerScoped = false + var head = cursor.next() + while let token = head { + switch sessionSQLMode(startingAt: token, cursor: &cursor, in: text, listIsServerScoped: &listIsServerScoped) { + case .literal(let mode): + result = !mode.uppercased().contains(noBackslashEscapes) + case .unknown: + result = false + case nil: + break + } + head = nextElementHead(&cursor) + } + return result + } + + private static func sessionSQLMode( + startingAt token: SQLTokenCursor.Token, + cursor: inout SQLTokenCursor, + in text: NSString, + listIsServerScoped: inout Bool + ) -> SQLModeAssignment? { + var current = token + while let word = current.word, sessionScopes.contains(word) || serverScopes.contains(word) { + listIsServerScoped = serverScopes.contains(word) + guard let next = cursor.next() else { return nil } + current = next + } + guard let name = current.identifier else { return nil } + switch name { + case sqlModeVariable: + guard !listIsServerScoped else { return nil } + case "@@" + sqlModeVariable: + break + case "@@SESSION", "@@LOCAL": + guard cursor.next()?.isSymbol(SQLTokenCursor.period) == true, + cursor.next()?.identifier == sqlModeVariable else { return nil } + default: + return nil + } + guard cursor.next()?.isSymbol(SQLTokenCursor.equals) == true, let value = cursor.next() else { return nil } + let range = cursor.tokenRange + switch value { + case .literal where range.length >= 2: + return .literal(text.substring(with: NSRange(location: range.location + 1, length: range.length - 2))) + case .word("DEFAULT"): + return .literal("") + default: + return .unknown + } + } + + private static func nextElementHead(_ cursor: inout SQLTokenCursor) -> SQLTokenCursor.Token? { + while let token = cursor.next() { + if token.isSymbol(SQLTokenCursor.comma), cursor.parenDepth == 0 { + return cursor.next() + } + } + return nil + } + + private static func introducerWord(in text: NSString, range: NSRange) -> Introducer? { + guard range.length > 1, text.character(at: range.location) == underscore else { return nil } + let name = text.substring(with: NSRange(location: range.location + 1, length: range.length - 1)).lowercased() + return Introducer(range: range, characterSet: name) + } + + private static func isStringQuote(_ character: UInt16) -> Bool { + character == SqlLexer.singleQuote || character == SqlLexer.doubleQuote + } + + private static func replacement( + for introducer: Introducer, + literal span: SQLNonCodeSpan.Span, + in text: NSString, + context: LiteralContext + ) throws -> Replacement? { + let body = text.substring(with: NSRange(location: span.start + 1, length: max(0, span.contentEnd - span.start - 1))) + guard !unicodeCharacterSets.contains(introducer.characterSet), + body.unicodeScalars.contains(where: { !$0.isASCII }) else { + return nil + } + guard introducer.characterSet == binaryCharacterSet else { + guard isFollowedByCollate(span.end, in: text, grammar: context.grammar) else { + return Replacement(range: introducer.range, text: "_utf8mb4") + } + let literal = text.substring(with: NSRange(location: span.start, length: span.end - span.start)) + let range = NSRange(location: introducer.range.location, length: span.end - introducer.range.location) + return Replacement(range: range, text: "CONVERT(_utf8mb4\(literal) USING \(introducer.characterSet))") + } + guard exactlyRecoverableEncodings.contains(context.encoding), + context.backslashEscapesAreOn || !body.contains("\\"), + let bytes = body.data(using: context.encoding) else { + throw SQLTranscodedLiteralError.unrecoverableBinaryLiteral + } + let quote = UInt8(truncatingIfNeeded: text.character(at: span.start)) + let hex = unescaped([UInt8](bytes), quote: quote).map { String(format: "%02X", $0) }.joined() + let range = NSRange(location: introducer.range.location, length: span.end - introducer.range.location) + return Replacement(range: range, text: "X'\(hex)'") + } + + private static func isFollowedByCollate(_ offset: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool { + var index = offset + while index < text.length { + if SqlLexer.isWhitespace(text.character(at: index)) { + index += 1 + continue + } + guard let comment = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar), comment.kind.isComment else { break } + index = max(comment.end, index + 1) + } + let keyword = "COLLATE" + let keywordLength = (keyword as NSString).length + guard index + keywordLength <= text.length, + text.substring(with: NSRange(location: index, length: keywordLength)).uppercased() == keyword else { + return false + } + return index + keywordLength == text.length + || !SQLNonCodeSpan.isWordUnit(text.character(at: index + keywordLength)) + } + + private static func unescaped(_ bytes: [UInt8], quote: UInt8) -> [UInt8] { + var result: [UInt8] = [] + result.reserveCapacity(bytes.count) + var index = 0 + while index < bytes.count { + let byte = bytes[index] + if byte == quote, index + 1 < bytes.count, bytes[index + 1] == quote { + result.append(quote) + index += 2 + continue + } + guard byte == backslash, index + 1 < bytes.count else { + result.append(byte) + index += 1 + continue + } + let escaped = bytes[index + 1] + if escaped == percent || escaped == UInt8(ascii: "_") { + result.append(contentsOf: [backslash, escaped]) + } else { + result.append(escapedBytes[escaped] ?? escaped) + } + index += 2 + } + return result + } +} diff --git a/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift b/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift index d9bf9fffa7..b5438a1efa 100644 --- a/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift +++ b/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift @@ -64,7 +64,7 @@ internal struct LinkedFileVersionHistoryProvider: VersionHistoryProvider { case .gitRevision(let commit, let path): let client = try makeClient() let data = try await gitCall { try await client.blob(revision: commit, path: path, in: directory) } - let workingCopyEncoding = TextEncodingAttribute.read(from: fileURL)?.encoding + let workingCopyEncoding = FileTextLoader.resolvedEncoding(of: fileURL) guard let content = FileTextLoader.decode(data, declaredEncoding: workingCopyEncoding) else { throw VersionHistoryError.undecodableContent } diff --git a/TablePro/Models/Export/ImportModels.swift b/TablePro/Models/Export/ImportModels.swift index a7c72f6315..e13ac772fa 100644 --- a/TablePro/Models/Export/ImportModels.swift +++ b/TablePro/Models/Export/ImportModels.swift @@ -6,6 +6,7 @@ // import Foundation +import TableProTabularIO // MARK: - Import Encoding Options @@ -23,9 +24,28 @@ enum ImportEncoding: String, CaseIterable, Identifiable { case latin1 = "Latin1" case windows1252 = "Windows-1252" case ascii = "ASCII" + case shiftJIS = "Shift_JIS" + case eucJP = "EUC-JP" + case gb18030 = "GB18030" + case big5 = "Big5" + case eucKR = "EUC-KR" var id: String { rawValue } + init?(detected: TabularTextEncoding) { + guard let match = Self.allCases.first(where: { $0.tabularEncoding == detected }) else { return nil } + self = match + } + + var canHideABackslashInsideACharacter: Bool { + switch self { + case .shiftJIS, .gb18030, .big5: + return true + case .utf8, .utf16, .utf16LittleEndian, .utf16BigEndian, .latin1, .windows1252, .ascii, .eucJP, .eucKR: + return false + } + } + var label: String { switch self { case .utf8: return "UTF-8" @@ -35,18 +55,33 @@ enum ImportEncoding: String, CaseIterable, Identifiable { case .latin1: return "Latin-1" case .windows1252: return "Windows-1252" case .ascii: return "ASCII" + case .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: + return tabularEncoding?.displayName ?? rawValue } } var encoding: String.Encoding { switch self { - case .utf8: return .utf8 case .utf16: return .utf16 + case .ascii: return .ascii + case .utf8, .utf16LittleEndian, .utf16BigEndian, .latin1, .windows1252, .shiftJIS, .eucJP, .gb18030, .big5, .eucKR: + return tabularEncoding?.foundationEncoding ?? .utf8 + } + } + + private var tabularEncoding: TabularTextEncoding? { + switch self { + case .utf8: return .utf8 case .utf16LittleEndian: return .utf16LittleEndian case .utf16BigEndian: return .utf16BigEndian case .latin1: return .isoLatin1 - case .windows1252: return .windowsCP1252 - case .ascii: return .ascii + case .windows1252: return .windows1252 + case .shiftJIS: return .shiftJIS + case .eucJP: return .eucJP + case .gb18030: return .gb18030 + case .big5: return .big5 + case .eucKR: return .eucKR + case .utf16, .ascii: return nil } } } diff --git a/TablePro/Models/Query/FileTabBaseline.swift b/TablePro/Models/Query/FileTabBaseline.swift index aabd259c75..c2fe14ea5e 100644 --- a/TablePro/Models/Query/FileTabBaseline.swift +++ b/TablePro/Models/Query/FileTabBaseline.swift @@ -17,8 +17,29 @@ import Foundation /// Every path that rebuilds a file-backed tab reads it back here, so there is one door. internal enum FileTabBaseline { internal static func hydrate(_ tab: inout QueryTab) { - guard let url = tab.content.sourceFileURL, let loaded = FileTextLoader.load(url) else { return } - record(loaded.content, stamp: loaded.stamp, encoding: loaded.textEncoding, in: &tab.content) + guard let url = tab.content.sourceFileURL else { return } + if let recorded = tab.content.sourceFileEncoding, let loaded = FileTextLoader.load(url, as: recorded) { + record(loaded.content, stamp: loaded.stamp, encoding: loaded.textEncoding, in: &tab.content) + return + } + guard let loaded = FileTextLoader.load(url) else { return } + guard tab.content.sourceFileEncoding == nil, readsDifferentlyThanLatin1(loaded), + let latin1 = FileTextLoader.load(url, as: FileTextEncoding(encoding: .isoLatin1)), + tab.content.query != loaded.content else { + record(loaded.content, stamp: loaded.stamp, encoding: loaded.textEncoding, in: &tab.content) + return + } + if tab.content.query == latin1.content { + adopt(loaded, into: &tab.content) + return + } + record(latin1.content, stamp: latin1.stamp, encoding: latin1.textEncoding, in: &tab.content) + } + + private static func readsDifferentlyThanLatin1(_ loaded: FileTextLoader.LoadedText) -> Bool { + let encoding = loaded.textEncoding + guard encoding.byteOrderMark == nil, encoding.attribute == nil else { return false } + return encoding.encoding != .utf8 && encoding.encoding != .isoLatin1 } internal static func hydrate(_ tabs: inout [QueryTab]) { diff --git a/TablePro/Models/Query/QueryTab.swift b/TablePro/Models/Query/QueryTab.swift index ebb4436e60..017d909e32 100644 --- a/TablePro/Models/Query/QueryTab.swift +++ b/TablePro/Models/Query/QueryTab.swift @@ -224,7 +224,8 @@ struct QueryTab: Identifiable, Equatable { self.content = TabQueryContent( query: persisted.query, queryParameters: persisted.queryParameters ?? [], - sourceFileURL: persisted.sourceFileURL + sourceFileURL: persisted.sourceFileURL, + sourceFileEncoding: persisted.sourceFileEncoding ) self.execution = TabExecutionState() self.tableContext = TabTableContext( @@ -412,6 +413,7 @@ struct QueryTab: Identifiable, Equatable { databaseName: tableContext.databaseName, schemaName: tableContext.schemaName, sourceFileURL: content.sourceFileURL, + sourceFileEncoding: content.sourceFileEncoding, erDiagramSchemaKey: display.erDiagramSchemaKey, objectRef: display.objectRef, versionHistorySubject: display.versionHistorySubject, diff --git a/TablePro/Models/Query/QueryTabState.swift b/TablePro/Models/Query/QueryTabState.swift index 34fd70239a..642e512ee0 100644 --- a/TablePro/Models/Query/QueryTabState.swift +++ b/TablePro/Models/Query/QueryTabState.swift @@ -40,6 +40,7 @@ struct PersistedTab: Codable { var databaseName: String = "" var schemaName: String? var sourceFileURL: URL? + var sourceFileEncoding: FileTextEncoding? var erDiagramSchemaKey: String? var objectRef: DatabaseObjectRef? var versionHistorySubject: VersionHistorySubject? @@ -72,6 +73,7 @@ struct PersistedTab: Codable { databaseName: String = "", schemaName: String? = nil, sourceFileURL: URL? = nil, + sourceFileEncoding: FileTextEncoding? = nil, erDiagramSchemaKey: String? = nil, objectRef: DatabaseObjectRef? = nil, versionHistorySubject: VersionHistorySubject? = nil, @@ -97,6 +99,7 @@ struct PersistedTab: Codable { self.databaseName = databaseName self.schemaName = schemaName self.sourceFileURL = sourceFileURL + self.sourceFileEncoding = sourceFileEncoding self.erDiagramSchemaKey = erDiagramSchemaKey self.objectRef = objectRef self.versionHistorySubject = versionHistorySubject @@ -115,7 +118,7 @@ struct PersistedTab: Codable { private enum CodingKeys: String, CodingKey { case id, title, query, tabType, tableName, isView, objectTypeRawValue, databaseName, schemaName - case sourceFileURL, erDiagramSchemaKey, objectRef, versionHistorySubject, queryParameters + case sourceFileURL, sourceFileEncoding, erDiagramSchemaKey, objectRef, versionHistorySubject, queryParameters case sortColumns, sortSource, restoredPage, restoredPageSize, cursorOffset, cursorLength case collapsedFoldRanges case columnWidths, columnContentWidths, windowGroupIndex @@ -134,6 +137,7 @@ struct PersistedTab: Codable { databaseName = try container.decodeIfPresent(String.self, forKey: .databaseName) ?? "" schemaName = try container.decodeIfPresent(String.self, forKey: .schemaName) sourceFileURL = try container.decodeIfPresent(URL.self, forKey: .sourceFileURL) + sourceFileEncoding = try container.decodeIfPresent(FileTextEncoding.self, forKey: .sourceFileEncoding) erDiagramSchemaKey = try container.decodeIfPresent(String.self, forKey: .erDiagramSchemaKey) objectRef = try container.decodeIfPresent(DatabaseObjectRef.self, forKey: .objectRef) versionHistorySubject = try container.decodeIfPresent(VersionHistorySubject.self, forKey: .versionHistorySubject) diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 4732c63b49..45eacd651e 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -130490,15 +130490,14 @@ } } }, - "Row %@, column %@ contains “%@”, which %@ cannot store. Choose Save As and pick UTF-8." : { - "localizations" : { - "en" : { - "stringUnit" : { - "state" : "new", - "value" : "Row %1$@, column %2$@ contains “%3$@”, which %4$@ cannot store. Choose Save As and pick UTF-8." - } - } - } + "Row %@, column “%@” contains %@, which %@ cannot store. Choose Save As and pick UTF-8." : { + + }, + "A row the filter hides holds %@ in column “%@”, which %@ cannot store. Clear the filter to find it, or choose Save As and pick UTF-8." : { + + }, + "The column name “%@” contains %@, which %@ cannot store. Rename the column, or choose Save As and pick UTF-8." : { + }, "Row %@, column %@ is not valid JSON. Fix the value or clear it, then save again." : { "localizations" : { @@ -182733,6 +182732,21 @@ }, "1 row has a different number of fields" : { + }, + "1 line has unreadable characters" : { + + }, + "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again." : { + + }, + "Line %@ is not valid %@ text. Choose the file's encoding in the import options, then import again." : { + + }, + "%@ lines have unreadable characters" : { + + }, + "Bytes on those lines are not valid %@ and show as �. Pick another encoding in File Properties." : { + }, "%2$lld rows do not fit a %1$@ INSERT on their own, so their statements pass that size." : { diff --git a/TablePro/Views/DataFiles/DataFileSheets.swift b/TablePro/Views/DataFiles/DataFileSheets.swift index 4ad31f6149..ceb04e5b6a 100644 --- a/TablePro/Views/DataFiles/DataFileSheets.swift +++ b/TablePro/Views/DataFiles/DataFileSheets.swift @@ -100,7 +100,7 @@ struct DataFilePropertiesSheet: View { } Picker("Encoding", selection: $encoding) { ForEach(DataFilePropertyOptions.encodings, id: \.self) { option in - Text(DataFileEncodingNames.name(for: option)).tag(option) + Text(option.displayName).tag(option) } } Picker("Line ending", selection: $lineEnding) { diff --git a/TablePro/Views/DataFiles/DataFileStatusBar.swift b/TablePro/Views/DataFiles/DataFileStatusBar.swift index 14fd0d53a3..7ca090bf5d 100644 --- a/TablePro/Views/DataFiles/DataFileStatusBar.swift +++ b/TablePro/Views/DataFiles/DataFileStatusBar.swift @@ -21,6 +21,14 @@ struct DataFileStatusBar: View { Text(DataFileCountPhrase.raggedRows(controller.raggedRowCount)) .help(String(localized: "Those rows have more or fewer fields than the first row. Missing fields read as empty.")) } + if controller.undecodableLineCount > 0, let dialect = controller.dialect { + Text(DataFileCountPhrase.undecodableLines(controller.undecodableLineCount)) + .help(String( + format: String(localized: "Bytes on those lines are not valid %@ and show as \u{FFFD}. Pick another encoding in File Properties."), + dialect.encoding.displayName + )) + .accessibilityIdentifier("data-file-undecodable-lines") + } } if let message = controller.statusMessage { Text(message) @@ -33,6 +41,8 @@ struct DataFileStatusBar: View { if let dialect = controller.dialect, controller.kind?.format == .delimited { Text(DataFileDialectDescription.summary(dialect)) .accessibilityLabel(String(localized: "File format")) + .accessibilityValue(DataFileDialectDescription.summary(dialect)) + .accessibilityIdentifier("data-file-format") } if controller.pageCount > 1 { pageControls @@ -100,7 +110,7 @@ struct DataFileStatusBar: View { enum DataFileDialectDescription { static func summary(_ dialect: DelimitedDialect) -> String { - [delimiterName(dialect.delimiter), DataFileEncodingNames.name(for: dialect.encoding), lineEndingName(dialect.lineEnding)] + [delimiterName(dialect.delimiter), dialect.encoding.displayName, lineEndingName(dialect.lineEnding)] .joined(separator: ", ") } diff --git a/TablePro/Views/Import/ImportDialog.swift b/TablePro/Views/Import/ImportDialog.swift index b04e9d510b..6a72091b14 100644 --- a/TablePro/Views/Import/ImportDialog.swift +++ b/TablePro/Views/Import/ImportDialog.swift @@ -34,6 +34,7 @@ struct ImportDialog: View { @State private var statementCount: Int = 0 @State private var isCountingStatements = false @State private var selectedEncoding: ImportEncoding + @State private var detectedEncoding: ImportEncoding? @State private var selectedFormatId: String = "sql" @State private var settingsSnapshot: PluginSettingsSnapshot? @State private var importSucceeded = false @@ -345,7 +346,9 @@ struct ImportDialog: View { private func recordSuccessfulImport() { importSucceeded = true - TransferDialogStorage.shared.saveLastImportEncoding(selectedEncoding) + if selectedEncoding != detectedEncoding { + TransferDialogStorage.shared.saveLastImportEncoding(selectedEncoding) + } settingsSnapshot = nil } @@ -434,6 +437,17 @@ struct ImportDialog: View { if let preview = decoder.decode(previewData) { filePreview = preview hasPreviewError = false + } else if let detected = SQLDumpEncodingChoice.encoding( + replacing: selectedEncoding, + forPreview: previewData, + isWholeFile: previewData.count < maxPreviewSize, + family: TransactionEngineFamily.of(connection.type), + grammar: connection.type.lexicalGrammar + ) { + Self.logger.info("Preview did not decode as \(selectedEncoding.rawValue, privacy: .public); switching to the detected \(detected.rawValue, privacy: .public)") + detectedEncoding = detected + selectedEncoding = detected + return } else { filePreview = String(format: String(localized: "Failed to load preview using encoding: %@. Try selecting a different text encoding."), selectedEncoding.label) hasPreviewError = true diff --git a/TableProMobile/TableProMobile/Intents/IntentDataError.swift b/TableProMobile/TableProMobile/Intents/IntentDataError.swift index d013fedbbe..1a9d26d7c7 100644 --- a/TableProMobile/TableProMobile/Intents/IntentDataError.swift +++ b/TableProMobile/TableProMobile/Intents/IntentDataError.swift @@ -11,7 +11,7 @@ nonisolated enum IntentDataError: Error, CustomLocalizedStringResourceConvertibl case unknownColumns([String], String) case emptyPayload case expectedSingleRow - case fileIsNotUTF8 + case unreadableText(line: Int, encoding: String) case invalidTextEncoding case jsonArrayContainsNonObject case invalidJSONShape @@ -39,8 +39,8 @@ nonisolated enum IntentDataError: Error, CustomLocalizedStringResourceConvertibl return "No data was provided to add." case .expectedSingleRow: return "Add Row to Table expects one row. Use Add Rows to Table for multiple rows." - case .fileIsNotUTF8: - return LocalizedStringResource("The file is not UTF-8 text.") + case .unreadableText(let line, let encoding): + return "Line \(line) of the file is not valid \(encoding) text." case .invalidTextEncoding: return LocalizedStringResource("The text encoding is invalid.") case .jsonArrayContainsNonObject: diff --git a/TableProMobile/TableProMobile/Intents/RowPayload.swift b/TableProMobile/TableProMobile/Intents/RowPayload.swift index 6169b4a2e5..2b9b5e23e1 100644 --- a/TableProMobile/TableProMobile/Intents/RowPayload.swift +++ b/TableProMobile/TableProMobile/Intents/RowPayload.swift @@ -1,5 +1,6 @@ import AppIntents import Foundation +import TableProTabularIO import UniformTypeIdentifiers nonisolated enum RowPayload { @@ -27,11 +28,7 @@ nonisolated enum RowPayload { private static func rawContent(data: String?, file: IntentFile?) async throws -> String { if let file { - let fileData = try await file.data(contentType: .data) - guard let text = String(data: fileData, encoding: .utf8) else { - throw IntentDataError.fileIsNotUTF8 - } - return text + return try text(of: try await file.data(contentType: .data)) } if let data, !data.isEmpty { return data @@ -39,6 +36,25 @@ nonisolated enum RowPayload { throw IntentDataError.emptyPayload } + static func text(of fileData: Data) throws -> String { + let sniff = TabularEncodingDetector.sniff(fileData) + guard sniff.encoding != .utf8 else { + if let line = TabularTextTranscoder.firstInvalidUTF8Line(in: fileData, skippingPrefix: sniff.byteOrderMarkLength) { + throw IntentDataError.unreadableText(line: line, encoding: sniff.encoding.displayName) + } + return TabularTextCodec.utf8String(fileData.dropFirst(sniff.byteOrderMarkLength)) + } + let decoded = try TabularTextTranscoder.utf8Data( + from: fileData, + encoding: sniff.encoding, + skippingPrefix: sniff.byteOrderMarkLength + ) + if let line = decoded.firstUndecodableLine { + throw IntentDataError.unreadableText(line: line, encoding: sniff.encoding.displayName) + } + return TabularTextCodec.utf8String(decoded.data) + } + static func parseJSON(_ text: String) throws -> [PayloadRow] { guard let data = text.data(using: .utf8) else { throw IntentDataError.invalidTextEncoding diff --git a/TableProMobile/TableProMobile/Localizable.xcstrings b/TableProMobile/TableProMobile/Localizable.xcstrings index 73846e7a96..be754acbac 100644 --- a/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/TableProMobile/TableProMobile/Localizable.xcstrings @@ -16680,30 +16680,30 @@ } } }, - "The file is not UTF-8 text." : { + "Line %lld of the file is not valid %@ text." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "파일이 UTF-8 텍스트가 아닙니다." + "value" : "파일의 %1$lld번째 줄이 올바른 %2$@ 텍스트가 아닙니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Tệp không phải văn bản UTF-8." + "value" : "Dòng %1$lld của tệp không phải văn bản %2$@ hợp lệ." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "该文件不是 UTF-8 文本。" + "value" : "文件第 %1$lld 行不是有效的 %2$@ 文本。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "該檔案不是 UTF-8 文字。" + "value" : "檔案第 %1$lld 行不是有效的 %2$@ 文字。" } } } diff --git a/TableProMobile/TableProMobileTests/Intents/RowPayloadTests.swift b/TableProMobile/TableProMobileTests/Intents/RowPayloadTests.swift index bdc29458c7..b93e4572ed 100644 --- a/TableProMobile/TableProMobileTests/Intents/RowPayloadTests.swift +++ b/TableProMobile/TableProMobileTests/Intents/RowPayloadTests.swift @@ -89,4 +89,27 @@ struct RowPayloadTests { _ = try RowPayload.parseCSV(",\nAda,36") } } + + @Test("reads a Shift JIS CSV file as its Japanese text") + func shiftJISFile() throws { + let csv = "氏名,住所\n山田太郎,東京都港区芝公園\n鈴木花子,大阪市北区梅田\n" + let data = try #require(csv.data(using: .shiftJIS)) + #expect(try RowPayload.text(of: data) == csv) + } + + @Test("reads a UTF-16 file and drops a UTF-8 byte order mark") + func markedFiles() throws { + let csv = "name,city\nAda,London\n" + let utf16 = Data([0xFF, 0xFE]) + (try #require(csv.data(using: .utf16LittleEndian))) + #expect(try RowPayload.text(of: utf16) == csv) + #expect(try RowPayload.text(of: Data([0xEF, 0xBB, 0xBF]) + Data(csv.utf8)) == csv) + } + + @Test("names the line a file cannot be read at") + func unreadableLine() throws { + let data = Data("name,city\nJürgen,Köln\nZoë,Paris\n".utf8) + Data([0x43, 0x61, 0x66, 0xE9, 0x0A]) + #expect(throws: IntentDataError.unreadableText(line: 4, encoding: "UTF-8")) { + _ = try RowPayload.text(of: data) + } + } } diff --git a/TableProMobile/project.yml b/TableProMobile/project.yml index 955ae14892..8d162b02d9 100644 --- a/TableProMobile/project.yml +++ b/TableProMobile/project.yml @@ -131,6 +131,7 @@ targets: - TableProSSHTransport - TableProSync - TableProSyncTransport + - TableProTabularIO - package: TableProOracle product: TableProOracleCore - framework: ../Libs/ios/DuckDB.xcframework diff --git a/TableProTests/Core/DataFiles/DataFileControllerTests.swift b/TableProTests/Core/DataFiles/DataFileControllerTests.swift index d7cebacd5e..a89e8ff46e 100644 --- a/TableProTests/Core/DataFiles/DataFileControllerTests.swift +++ b/TableProTests/Core/DataFiles/DataFileControllerTests.swift @@ -486,7 +486,158 @@ struct DataFileControllerTests { controller.saveEncoding = .utf8 try controller.write(to: output, typeName: DataFileKind.commaSeparatedType) #expect(try String(contentsOf: output, encoding: .utf8) == "n\n漢\n") - controller.adoptSaveEncoding() + controller.adoptWrittenDialect() #expect(controller.dialect?.encoding == .utf8) } + + private static let japaneseExport = """ + 商品コード,商品名,在庫管理区分,ｻﾞｲｺｶﾝﾘｸﾌﾞﾝ + 10001,ソフトウェア保守,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ + 10002,ポイント交換,在庫管理しない,ｻﾞｲｺｶﾝﾘｼﾅｲ + 10003,髙橋商店の注文,在庫管理する,ｻﾞｲｺｶﾝﾘｽﾙ + + """ + + private func shiftJIS(_ text: String) throws -> Data { + try #require(text.data(using: .shiftJIS, allowLossyConversion: false)) + } + + @Test("A Shift JIS export from Excel opens with its Japanese text") + func shiftJISOpensReadable() async throws { + let (controller, _) = try await loaded(data: try shiftJIS(Self.japaneseExport), fileExtension: "csv") + #expect(controller.dialect?.encoding == .shiftJIS) + #expect(controller.columnNames.displayNames == ["商品コード", "商品名", "在庫管理区分", "ｻﾞｲｺｶﾝﾘｸﾌﾞﾝ"]) + #expect(column(controller, 1) == ["ソフトウェア保守", "ポイント交換", "髙橋商店の注文"]) + #expect(controller.undecodableLineCount == 0) + } + + @Test("A byte the encoding cannot read shows as a replacement character and is counted") + func undecodableBytesAreCounted() async throws { + var bytes = try shiftJIS(Self.japaneseExport) + bytes.append(contentsOf: Array("10004,".utf8) + [0x82] + Array(",x,y\n".utf8)) + let (controller, _) = try await loaded(data: bytes, fileExtension: "csv") + #expect(controller.dialect?.encoding == .shiftJIS) + #expect(controller.undecodableLineCount == 1) + #expect(column(controller, 1).last == "\u{FFFD}") + } + + @Test("Saving an edited Shift JIS file keeps the untouched rows' bytes") + func shiftJISSaveKeepsUntouchedRows() async throws { + let original = try shiftJIS(Self.japaneseExport.replacingOccurrences(of: "\n", with: "\r\n")) + let (controller, url) = try await loaded(data: original, fileExtension: "csv") + controller.setCell(pageRow: 1, column: 1, text: "ポイント還元") + let output = url.deletingLastPathComponent().appendingPathComponent("out.csv") + try controller.write(to: output, typeName: DataFileKind.commaSeparatedType) + let expected = try shiftJIS( + Self.japaneseExport.replacingOccurrences(of: "ポイント交換", with: "ポイント還元") + .replacingOccurrences(of: "\n", with: "\r\n") + ) + #expect(try Data(contentsOf: output) == expected) + } + + @Test("Reopening the file keeps the encoding chosen in File Properties") + func reloadKeepsTheChosenEncoding() async throws { + let (controller, url) = try await loaded(data: try shiftJIS(Self.japaneseExport), fileExtension: "csv") + let kind = try #require(DataFileKind.classify(url)) + var chosen = try #require(controller.dialect) + chosen.encoding = .windows1252 + controller.load(url: url, kind: kind, dialectOverride: chosen) + await controller.waitForPendingWork() + #expect(controller.dialect?.encoding == .windows1252) + controller.load(url: url, kind: kind) + await controller.waitForPendingWork() + #expect(controller.loadState == .loaded) + #expect(controller.dialect?.encoding == .windows1252) + } + + @Test("Save As to a tab-separated file makes a later reload read tabs, even after a File Properties choice") + func saveAsAdoptsTheWrittenDialect() async throws { + let (controller, url) = try await loaded(data: try shiftJIS(Self.japaneseExport), fileExtension: "csv") + let csvKind = try #require(DataFileKind.classify(url)) + var chosen = try #require(controller.dialect) + chosen.encoding = .shiftJIS + controller.load(url: url, kind: csvKind, dialectOverride: chosen) + await controller.waitForPendingWork() + + let output = url.deletingLastPathComponent().appendingPathComponent("people.tsv") + try controller.write(to: output, typeName: DataFileKind.tabSeparatedType) + controller.adoptWrittenDialect() + let tsvKind = try #require(DataFileKind.classify(output)) + controller.load(url: output, kind: tsvKind) + await controller.waitForPendingWork() + + #expect(controller.dialect?.delimiter == DelimitedDialect.tab) + #expect(controller.dialect?.encoding == .shiftJIS) + #expect(column(controller, 1) == ["ソフトウェア保守", "ポイント交換", "髙橋商店の注文"]) + } + + @Test("Switching the header off after a File Properties choice survives a reload") + func headerSwitchSurvivesAReload() async throws { + let (controller, url) = try await loaded(data: try shiftJIS(Self.japaneseExport), fileExtension: "csv") + let kind = try #require(DataFileKind.classify(url)) + let chosen = try #require(controller.dialect) + controller.load(url: url, kind: kind, dialectOverride: chosen) + await controller.waitForPendingWork() + controller.setUsesFirstRowAsHeader(false) + await controller.waitForPendingWork() + controller.load(url: url, kind: kind) + await controller.waitForPendingWork() + #expect(controller.dialect?.hasHeaderRow == false) + #expect(controller.totalRowCount == 4) + } + + @Test("Saving in another encoding clears the unreadable-line count measured for the old one") + func saveAsInAnotherEncodingClearsTheCount() async throws { + var bytes = try shiftJIS(Self.japaneseExport) + bytes.append(contentsOf: Array("10004,".utf8) + [0x82] + Array(",x,y\n".utf8)) + let (controller, url) = try await loaded(data: bytes, fileExtension: "csv") + #expect(controller.undecodableLineCount == 1) + controller.saveEncoding = .utf8 + try controller.write(to: url.deletingLastPathComponent().appendingPathComponent("out.csv"), typeName: DataFileKind.commaSeparatedType) + controller.adoptWrittenDialect() + #expect(controller.dialect?.encoding == .utf8) + #expect(controller.undecodableLineCount == 0) + } + + @Test("The working copy is a copy, so a change to the original does not reach an open file") + func workingCopyIsAlwaysACopy() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("DataFileWorkingCopyTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let url = directory.appendingPathComponent("people.csv") + try Data("a,b\n1,2\n".utf8).write(to: url) + let workingCopy = try DataFileWorkingCopy() + let snapshot = try await workingCopy.snapshot(of: url, kind: try #require(DataFileKind.classify(url))) + #expect(snapshot != url) + try Data("changed\n".utf8).write(to: url) + #expect(try Data(contentsOf: snapshot) == Data("a,b\n1,2\n".utf8)) + } + + @Test("The cannot-store error names the row the grid shows, the column by name, and selects the row") + func unencodableErrorNamesTheVisibleRow() async throws { + let bytes = Data([0x6E, 0x61, 0x6D, 0x65, 0x2C, 0x6E, 0x0A, 0x7A, 0x6F, 0x65, 0x2C, 0x31, 0x0A] + + [0x63, 0x61, 0x66, 0xE9, 0x2C, 0x32, 0x0A, 0x61, 0x70, 0x70, 0x6C, 0x65, 0x2C, 0x33, 0x0A]) + let (controller, url) = try await loaded(data: bytes, fileExtension: "csv") + #expect(controller.dialect?.encoding == .windows1252) + controller.updateSort(SortState(columns: [SortColumn(columnIndex: 0, direction: .ascending)])) + await controller.waitForPendingWork() + #expect(column(controller, 0) == ["apple", "caf\u{E9}", "zoe"]) + controller.setCell(pageRow: 1, column: 0, text: "漢") + controller.selectedRowIndices = [] + let output = url.deletingLastPathComponent().appendingPathComponent("out.csv") + #expect { + try controller.write(to: output, typeName: DataFileKind.commaSeparatedType) + } throws: { error in + let message = error.localizedDescription + return message.contains("Row 2") && message.contains("“name”") && message.contains("“漢”") + } + #expect(controller.selectedRowIndices == [1]) + } + + @Test("An invisible character is named by its code point") + func invisibleCharactersAreNamed() { + #expect(DataFileController.describe("\u{A0}") == "U+00A0 NO-BREAK SPACE") + #expect(DataFileController.describe("漢") == "“漢”") + } } diff --git a/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift b/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift new file mode 100644 index 0000000000..3810121510 --- /dev/null +++ b/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift @@ -0,0 +1,88 @@ +// +// SqlFileImportSourceEncodingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTabularIO +import Testing + +struct SqlFileImportSourceEncodingTests { + private func statements(of bytes: Data, encoding: String.Encoding, family: TransactionEngineFamily) async throws -> [String] { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SqlFileImportSourceEncodingTests-\(UUID().uuidString).sql") + try bytes.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let grammar = family == .mysql ? TestGrammar.mysql : TestGrammar.postgres + let source = SqlFileImportSource(url: url, encoding: encoding, grammar: grammar, family: family) + var result: [String] = [] + for try await (statement, _) in try await source.statements() { + result.append(statement) + } + return result + } + + @Test("A Shift JIS mysqldump keeps the session's UTF-8 client character set and its text decoded") + func shiftJISMysqldump() async throws { + let dump = "/*!40101 SET NAMES cp932 */;\nINSERT INTO t VALUES ('日本語 髙橋');\n" + let bytes = try #require(dump.data(using: .shiftJIS, allowLossyConversion: false)) + let result = try await statements(of: bytes, encoding: ImportEncoding.shiftJIS.encoding, family: .mysql) + #expect(result == [ + "/*!40101 SET character_set_client = @@session.character_set_client */", + "INSERT INTO t VALUES ('日本語 髙橋')" + ]) + } + + @Test("A Shift JIS pg_dump runs with a UTF-8 client encoding") + func shiftJISPgDump() async throws { + let dump = "SET client_encoding = 'SJIS';\nINSERT INTO t VALUES ('日本語');\n" + let bytes = try #require(dump.data(using: .shiftJIS, allowLossyConversion: false)) + let result = try await statements(of: bytes, encoding: ImportEncoding.shiftJIS.encoding, family: .postgres) + #expect(result.first == "SET client_encoding = 'UTF8'") + } + + @Test("A UTF-8 file keeps its own character set statement, since its bytes go to the server unchanged") + func utf8FileKeepsItsDeclaration() async throws { + let dump = "/*!40101 SET NAMES latin1 */;\nINSERT INTO t VALUES ('é');\n" + let result = try await statements(of: Data(dump.utf8), encoding: .utf8, family: .mysql) + #expect(result.first == "/*!40101 SET NAMES latin1 */") + } + + @Test("Each East Asian import encoding matches the detector's encoding") + func importEncodingsFollowTheDetector() { + let pairs: [(ImportEncoding, TabularTextEncoding)] = [ + (.shiftJIS, .shiftJIS), (.eucJP, .eucJP), (.gb18030, .gb18030), (.big5, .big5), (.eucKR, .eucKR), + (.utf8, .utf8), (.utf16LittleEndian, .utf16LittleEndian), (.windows1252, .windows1252), (.latin1, .isoLatin1) + ] + for (option, tabular) in pairs { + #expect(ImportEncoding(detected: tabular) == option) + #expect(option.encoding == tabular.foundationEncoding) + } + #expect(ImportEncoding.shiftJIS.rawValue == "Shift_JIS") + #expect(ImportEncoding.shiftJIS.label == "Shift JIS") + } + + @Test("A Latin-1 mysqldump keeps its binary column bytes and its accented text") + func latin1MysqldumpWithBinaryData() async throws { + let bytes = Array("/*!40101 SET NAMES latin1 */;\nINSERT INTO t VALUES ('Caf".utf8) + [0xE9] + + Array("', _binary'".utf8) + [0x00, 0xFF] + Array("');\n".utf8) + let result = try await statements(of: Data(bytes), encoding: .isoLatin1, family: .mysql) + #expect(result == [ + "/*!40101 SET character_set_client = @@session.character_set_client */", + "INSERT INTO t VALUES ('Café', X'00FF')" + ]) + } + + @Test("A Shift JIS dump holding binary data stops at that line instead of storing different bytes") + func shiftJISMysqldumpWithBinaryDataIsRefused() async throws { + let dump = "INSERT INTO t VALUES (1);\nINSERT INTO t VALUES (_binary'髙');\n" + let bytes = try #require(dump.data(using: .shiftJIS)) + await #expect(throws: SqlFileImportError.unrecoverableBinaryLiteral( + line: 2, + encoding: String.localizedName(of: .shiftJIS) + )) { + _ = try await statements(of: bytes, encoding: .shiftJIS, family: .mysql) + } + } +} diff --git a/TableProTests/Core/Utilities/FileTextLoaderTests.swift b/TableProTests/Core/Utilities/FileTextLoaderTests.swift index c151f3c519..bac8d790ae 100644 --- a/TableProTests/Core/Utilities/FileTextLoaderTests.swift +++ b/TableProTests/Core/Utilities/FileTextLoaderTests.swift @@ -242,4 +242,68 @@ struct FileTextLoaderTests { #expect(loaded.textEncoding == .utf8) #expect(header.textEncoding == .utf8) } + + private static let japaneseScript = """ + -- @name: 在庫一覧 + SELECT 商品コード, 商品名 FROM 在庫 WHERE 区分 = 'ｻﾞｲｺｶﾝﾘｽﾙ'; + INSERT INTO 顧客 (氏名, 住所) VALUES ('髙橋 太郎', '東京都港区芝公園４－２－８'); + UPDATE 顧客 SET 備考 = 'ポイント交換済み' WHERE 氏名 = '山﨑 花子'; + + """ + + @Test("A Shift JIS script with no recorded encoding opens as Shift JIS") + func readsAnUnmarkedShiftJISFile() throws { + let bytes = try #require(Self.japaneseScript.data(using: .shiftJIS, allowLossyConversion: false)) + + let loaded = try #require(try load(bytes)) + let header = try #require(try loadHeader(of: bytes)) + + #expect(loaded.encoding == .shiftJIS) + #expect(loaded.content == Self.japaneseScript) + #expect(header.encoding == .shiftJIS) + #expect(SQLFrontmatter.parse(header.content).name == "在庫一覧") + } + + @Test("A UTF-16 script without a byte order mark opens as UTF-16") + func readsAnUnmarkedUTF16File() throws { + let bytes = try #require(Self.japaneseScript.data(using: .utf16LittleEndian)) + + let loaded = try #require(try load(bytes)) + + #expect(loaded.encoding == .utf16LittleEndian) + #expect(loaded.content == Self.japaneseScript) + } + + @Test("A Western script with no recorded encoding still opens as ISO Latin-1") + func westernBytesStayLatin1() throws { + let bytes = try #require("-- @name: Caf\u{E9}\nSELECT 'cr\u{E8}me br\u{FB}l\u{E9}e';\n".data(using: .isoLatin1)) + + let loaded = try #require(try load(bytes)) + + #expect(loaded.encoding == .isoLatin1) + } + + @Test("A UTF-16 script of plain ASCII with no byte order mark opens as UTF-16, header and body alike") + func readsAnUnmarkedASCIIUTF16File() throws { + let text = "-- @name: Report\nSELECT id, name FROM customers;\nINSERT INTO t VALUES (1, 'abc');\n" + for encoding in [String.Encoding.utf16LittleEndian, .utf16BigEndian] { + let bytes = try #require(text.data(using: encoding)) + + let loaded = try #require(try load(bytes)) + let header = try #require(try loadHeader(of: bytes)) + + #expect(loaded.encoding == encoding) + #expect(loaded.content == text) + #expect(header.encoding == encoding) + #expect(SQLFrontmatter.parse(header.content).name == "Report") + } + } + + @Test("Version history reads an older commit in the encoding the working copy resolved to") + func resolvedEncodingFollowsDetection() throws { + let bytes = try #require(Self.japaneseScript.data(using: .shiftJIS, allowLossyConversion: false)) + let encoding = try withFile(bytes) { FileTextLoader.resolvedEncoding(of: $0) } + #expect(encoding == .shiftJIS) + #expect(FileTextLoader.decode(bytes, declaredEncoding: encoding) == Self.japaneseScript) + } } diff --git a/TableProTests/Core/Utilities/FileTextWriterTests.swift b/TableProTests/Core/Utilities/FileTextWriterTests.swift index 2384696c71..176cb0feb7 100644 --- a/TableProTests/Core/Utilities/FileTextWriterTests.swift +++ b/TableProTests/Core/Utilities/FileTextWriterTests.swift @@ -154,4 +154,13 @@ struct FileTextWriterTests { #expect(reloaded.encoding == .shiftJIS) #expect(reloaded.content == fixture.original) } + + @Test("A yen sign in a Shift JIS or EUC-JP file saves as the byte Japanese Windows shows as ¥") + func yenSignSavesAsJISRoman() throws { + let expected = try #require("SELECT '\\100~';\n".data(using: .shiftJIS)) + #expect(FileTextEncoding(encoding: .shiftJIS).bytes(of: "SELECT '¥100‾';\n") == expected) + let euc = try #require("SELECT '\\100';\n".data(using: .japaneseEUC)) + #expect(FileTextEncoding(encoding: .japaneseEUC).bytes(of: "SELECT '¥100';\n") == euc) + #expect(FileTextEncoding(encoding: .isoLatin1).bytes(of: "¥") == Data([0xA5])) + } } diff --git a/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift b/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift new file mode 100644 index 0000000000..dae26ab968 --- /dev/null +++ b/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift @@ -0,0 +1,80 @@ +// +// SQLClientEncodingRewriterTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct SQLClientEncodingRewriterTests { + private func mysql(_ statement: String) -> String? { + SQLClientEncodingRewriter.rewritten(statement, family: .mysql, grammar: TestGrammar.mysql) + } + + private func postgres(_ statement: String) -> String? { + SQLClientEncodingRewriter.rewritten(statement, family: .postgres, grammar: TestGrammar.postgres) + } + + private static let keepsSession = "character_set_client = @@session.character_set_client" + + @Test("A mysqldump character set statement keeps the session's own character set instead") + func mysqldumpSetNames() { + #expect(mysql("/*!40101 SET NAMES cp932 */") == "/*!40101 SET \(Self.keepsSession) */") + #expect(mysql("SET NAMES 'sjis'") == "SET \(Self.keepsSession)") + #expect(mysql("SET NAMES ujis COLLATE ujis_japanese_ci") == "SET \(Self.keepsSession)") + #expect(mysql("SET CHARACTER SET gbk") == "SET \(Self.keepsSession)") + #expect(mysql("SET CHARSET latin1") == "SET \(Self.keepsSession)") + } + + @Test("A routine section's client character set keeps the session's own instead") + func mysqldumpRoutineCharacterSet() { + #expect(mysql("/*!50003 SET character_set_client = cp932 */") + == "/*!50003 SET character_set_client = @@session.character_set_client */") + #expect(mysql("SET @@session.character_set_client = 'euckr'") + == "SET @@session.character_set_client = @@session.character_set_client") + #expect(mysql("SET SESSION character_set_client = big5, time_zone = '+00:00'") + == "SET SESSION character_set_client = @@session.character_set_client, time_zone = '+00:00'") + } + + @Test("Statements that already say UTF-8, restore a saved value, or change the server are left alone") + func mysqlStatementsLeftAlone() { + #expect(mysql("/*!50503 SET NAMES utf8mb4 */") == nil) + #expect(mysql("SET NAMES utf8") == nil) + #expect(mysql("SET NAMES DEFAULT") == nil) + #expect(mysql("/*!50003 SET character_set_client = @saved_cs_client */") == nil) + #expect(mysql("SET GLOBAL character_set_client = cp932") == nil) + #expect(mysql("SET @@global.character_set_client = cp932") == nil) + #expect(mysql("SET @@session.character_set_client = 33") == nil) + #expect(mysql("SET time_zone = '+00:00'") == nil) + #expect(mysql("INSERT INTO t VALUES ('SET NAMES cp932')") == nil) + } + + @Test("A GLOBAL or PERSIST scope carries to the list's later assignments") + func mysqlServerScopeCarriesAcrossTheList() { + #expect(mysql("SET GLOBAL max_connections = 1000, character_set_client = cp932") == nil) + #expect(mysql("SET PERSIST sql_mode = '', character_set_client = cp932") == nil) + #expect(mysql("SET PERSIST_ONLY a = 1, character_set_client = cp932") == nil) + #expect(mysql("SET GLOBAL a = 1, SESSION character_set_client = cp932") + == "SET GLOBAL a = 1, SESSION character_set_client = @@session.character_set_client") + #expect(mysql("SET GLOBAL a = 1, @@session.character_set_client = cp932") + == "SET GLOBAL a = 1, @@session.character_set_client = @@session.character_set_client") + } + + @Test("A pg_dump client encoding declares UTF-8 instead") + func pgDumpClientEncoding() { + #expect(postgres("SET client_encoding = 'SJIS'") == "SET client_encoding = 'UTF8'") + #expect(postgres("SET client_encoding TO 'EUC_JP'") == "SET client_encoding TO 'UTF8'") + #expect(postgres("SET SESSION client_encoding = BIG5") == "SET SESSION client_encoding = 'UTF8'") + #expect(postgres("SET NAMES 'LATIN1'") == "SET NAMES 'UTF8'") + #expect(postgres("SET client_encoding = 'UTF8'") == nil) + #expect(postgres("SET client_encoding = 'unicode'") == nil) + #expect(postgres("SET standard_conforming_strings = on") == nil) + } + + @Test("Engines without a client encoding statement are left alone") + func otherEnginesLeftAlone() { + #expect(SQLClientEncodingRewriter.rewritten("SET NAMES cp932", family: .sqlite, grammar: TestGrammar.sqlite) == nil) + #expect(SQLClientEncodingRewriter.rewritten("SET client_encoding = 'SJIS'", family: .sqlServer, grammar: TestGrammar.sqlServer) == nil) + } +} diff --git a/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift b/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift new file mode 100644 index 0000000000..070c13d1a8 --- /dev/null +++ b/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift @@ -0,0 +1,69 @@ +// +// SQLDumpEncodingChoiceTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTabularIO +import Testing + +struct SQLDumpEncodingChoiceTests { + private func choice(_ bytes: Data, replacing selected: ImportEncoding = .utf8, family: TransactionEngineFamily = .mysql) -> ImportEncoding? { + SQLDumpEncodingChoice.encoding( + replacing: selected, + forPreview: bytes, + isWholeFile: true, + family: family, + grammar: family == .mysql ? TestGrammar.mysql : TestGrammar.postgres + ) + } + + private func shiftJIS(_ text: String) throws -> Data { + try #require(text.data(using: .shiftJIS, allowLossyConversion: false)) + } + + @Test("A Latin-1 dump whose data reads as Shift JIS is read as the Latin-1 it declares") + func declaredLatin1OutranksDataThatLooksJapanese() throws { + var bytes = Data("-- MySQL dump 10.13\n/*!40101 SET NAMES latin1 */;\nINSERT INTO `users` VALUES (1,'".utf8) + bytes.append(try shiftJIS("日本語テストのデータです")) + bytes.append(contentsOf: Array("'),(2,'".utf8) + [0xE9, 0x5C, 0x27] + Array("); DROP TABLE users; -- '),(3,'bob');\n".utf8)) + #expect(TabularEncodingDetector.sniff(bytes).encoding == .shiftJIS) + #expect(choice(bytes).map { !$0.canHideABackslashInsideACharacter } ?? true) + + let latin1 = Data([0x2F, 0x2A, 0x21, 0x34, 0x30, 0x31, 0x30, 0x31] + Array(" SET NAMES latin1 */;\nINSERT INTO t VALUES ('Caf".utf8) + + [0xE9, 0x5C, 0x27] + Array("s');\n".utf8)) + #expect(choice(latin1) == .windows1252) + } + + @Test("Without a declaration, an encoding that can hide a backslash inside a character is never picked for you") + func undeclaredShiftJISIsNotPicked() throws { + var bytes = Data("INSERT INTO t VALUES ('/*!40101 SET NAMES sjis */', '".utf8) + bytes.append(try shiftJIS("日本語テストのデータです")) + bytes.append(contentsOf: Array("');\n".utf8)) + #expect(choice(bytes) == nil) + } + + @Test("A Shift JIS mysqldump or pg_dump that says so is read as Shift JIS") + func declaredShiftJISIsPicked() throws { + let mysql = try shiftJIS("/*!40101 SET NAMES cp932 */;\nINSERT INTO t VALUES ('日本語');\n") + #expect(choice(mysql) == .shiftJIS) + let postgres = try shiftJIS("SET client_encoding = 'SJIS';\nINSERT INTO t VALUES ('日本語');\n") + #expect(choice(postgres, family: .postgres) == .shiftJIS) + } + + @Test("An undeclared EUC-JP or Latin-1 script is read in the encoding it looks like") + func undeclaredSafeEncodingsArePicked() throws { + let euc = try #require("INSERT INTO t VALUES ('日本語のテキストです', '東京都港区');\n".data(using: .japaneseEUC)) + #expect(choice(euc) == .eucJP) + let western = try #require("INSERT INTO t VALUES ('Müller', 'Straße', 'Café crème');\n".data(using: .windowsCP1252)) + #expect(choice(western) == .windows1252) + } + + @Test("A declaration matching the chosen encoding changes nothing") + func declarationMatchingTheChoice() throws { + let bytes = Data("/*!40101 SET NAMES utf8mb4 */;\nINSERT INTO t VALUES ('x');\n".utf8) + #expect(choice(bytes) == nil) + #expect(SQLDumpEncodingChoice.declaredEncoding(in: bytes, family: .mysql, grammar: TestGrammar.mysql) == .utf8) + } +} diff --git a/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift b/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift new file mode 100644 index 0000000000..8a1400ea84 --- /dev/null +++ b/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift @@ -0,0 +1,105 @@ +// +// SQLTranscodedLiteralRewriterTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct SQLTranscodedLiteralRewriterTests { + private func rewritten(_ bytes: [UInt8], as encoding: String.Encoding) throws -> String? { + let statement = try #require(String(data: Data(bytes), encoding: encoding)) + return try SQLTranscodedLiteralRewriter.rewritten(statement, decodedFrom: encoding, grammar: TestGrammar.mysql) + } + + @Test("A binary literal read from a Latin-1 dump is sent as the exact bytes it held") + func binaryLiteralFromLatin1BecomesHex() throws { + let bytes = Array("INSERT INTO t VALUES (_binary'".utf8) + [0xE9, 0xFF, 0x5C, 0x30, 0x5C, 0x27, 0x27, 0x27, 0x41] + + Array("',1)".utf8) + #expect(try rewritten(bytes, as: .isoLatin1) == "INSERT INTO t VALUES (X'E9FF00272741',1)") + } + + @Test("A binary literal from a Windows-1252 dump keeps its bytes, backslash escapes decoded") + func binaryLiteralFromWindows1252BecomesHex() throws { + let bytes = Array("/*!40101 INSERT INTO t VALUES (_binary '".utf8) + [0x80, 0x5C, 0x6E, 0x5C, 0x25] + + Array("') */".utf8) + #expect(try rewritten(bytes, as: .windowsCP1252) == "/*!40101 INSERT INTO t VALUES (X'800A5C25') */") + } + + @Test("A binary literal that cannot be read back exactly stops the import") + func binaryLiteralFromShiftJISIsRefused() throws { + let statement = "INSERT INTO t VALUES (_binary'髙橋')" + #expect(throws: SQLTranscodedLiteralError.unrecoverableBinaryLiteral) { + try SQLTranscodedLiteralRewriter.rewritten(statement, decodedFrom: .shiftJIS, grammar: TestGrammar.mysql) + } + } + + @Test("A text introducer on a non-ASCII literal names the UTF-8 the import sends") + func textIntroducerBecomesUTF8() throws { + #expect(try SQLTranscodedLiteralRewriter.rewritten( + "SELECT _latin1'Café', _sjis \"東京\"", + decodedFrom: .shiftJIS, + grammar: TestGrammar.mysql + ) == "SELECT _utf8mb4'Café', _utf8mb4 \"東京\"") + #expect(try SQLTranscodedLiteralRewriter.rewritten( + "SELECT _latin1'Café' COLLATE latin1_german2_ci", + decodedFrom: .isoLatin1, + grammar: TestGrammar.mysql + ) == "SELECT CONVERT(_utf8mb4'Café' USING latin1) COLLATE latin1_german2_ci") + } + + @Test("ASCII bodies, Unicode introducers, hex literals and plain strings are left alone") + func literalsLeftAlone() throws { + let statements = [ + "INSERT INTO t VALUES (_binary'abc\\0', _utf8mb4'日本', _latin1 X'E9', X'E9', 'é')", + "SELECT my_column FROM t_1 WHERE name = 'Café'", + "SELECT `_binary` FROM t WHERE x = '_latin1'" + ] + for statement in statements { + #expect(try SQLTranscodedLiteralRewriter.rewritten(statement, decodedFrom: .isoLatin1, grammar: TestGrammar.mysql) == nil) + } + } + + @Test("A dump that turns backslash escapes off makes a binary literal holding a backslash stop the import") + func binaryLiteralWithBackslashNeedsKnownEscapes() throws { + let statement = "INSERT INTO t VALUES (_binary'\u{E9}\\n')" + #expect(throws: SQLTranscodedLiteralError.unrecoverableBinaryLiteral) { + try SQLTranscodedLiteralRewriter.rewritten( + statement, + decodedFrom: .isoLatin1, + backslashEscapesAreOn: false, + grammar: TestGrammar.mysql + ) + } + #expect(try SQLTranscodedLiteralRewriter.rewritten( + "INSERT INTO t VALUES (_binary'\u{E9}''x')", + decodedFrom: .isoLatin1, + backslashEscapesAreOn: false, + grammar: TestGrammar.mysql + ) == "INSERT INTO t VALUES (X'E92778')") + } + + @Test("The session's sql_mode decides whether backslashes escape") + func sqlModeTracking() { + func after(_ statement: String, _ isOn: Bool) -> Bool { + SQLTranscodedLiteralRewriter.backslashEscapesAreOn(after: statement, currently: isOn, grammar: TestGrammar.mysql) + } + #expect(after("/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */", true)) + #expect(!after("SET sql_mode = 'ANSI_QUOTES,NO_BACKSLASH_ESCAPES'", true)) + #expect(!after("SET SESSION sql_mode = @OLD_SQL_MODE", true)) + #expect(after("SET GLOBAL sql_mode = 'NO_BACKSLASH_ESCAPES'", true)) + #expect(!after("SET @@session.sql_mode = 'no_backslash_escapes'", true)) + #expect(after("SET sql_mode = DEFAULT", false)) + #expect(after("INSERT INTO t VALUES ('sql_mode')", true)) + } + + @Test("A comment between the literal and COLLATE still keeps the collation valid") + func collateAfterAComment() throws { + #expect(try SQLTranscodedLiteralRewriter.rewritten( + "SELECT _latin1'Café' /* note */ COLLATE latin1_bin", + decodedFrom: .isoLatin1, + grammar: TestGrammar.mysql + ) == "SELECT CONVERT(_utf8mb4'Café' USING latin1) /* note */ COLLATE latin1_bin") + } +} diff --git a/TableProTests/Models/FileTabBaselineTests.swift b/TableProTests/Models/FileTabBaselineTests.swift index 96c6a1b69e..c5f3a45f21 100644 --- a/TableProTests/Models/FileTabBaselineTests.swift +++ b/TableProTests/Models/FileTabBaselineTests.swift @@ -132,4 +132,83 @@ struct FileTabBaselineTests { #expect(tab.content.sourceFileEncoding == .utf8) } + + private static let japaneseScript = "SELECT 氏名 FROM 顧客 WHERE 住所 = '東京都港区';\nINSERT INTO 顧客 VALUES ('髙橋 太郎', 'ｻﾞｲｺｶﾝﾘ');\n" + + private func makeShiftJISFile() throws -> (URL, Data) { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("baseline-\(UUID().uuidString).sql") + let bytes = try #require(Self.japaneseScript.data(using: .shiftJIS, allowLossyConversion: false)) + try bytes.write(to: url) + return (url, bytes) + } + + @Test("A restored tab compares itself with its file read in the encoding it was saved with") + func hydratesInTheRecordedEncoding() throws { + let (url, _) = try makeShiftJISFile() + defer { try? FileManager.default.removeItem(at: url) } + var tab = fileTab(query: Self.japaneseScript, url: url) + tab.content.sourceFileEncoding = FileTextEncoding(encoding: .shiftJIS) + + FileTabBaseline.hydrate(&tab) + + #expect(tab.content.isFileDirty == false) + #expect(tab.content.sourceFileEncoding?.encoding == .shiftJIS) + } + + @Test("An older unedited tab that read a Shift JIS file as Latin-1 comes back as the Japanese text") + func olderCleanTabAdoptsTheNewReading() throws { + let (url, bytes) = try makeShiftJISFile() + defer { try? FileManager.default.removeItem(at: url) } + let latin1 = try #require(String(data: bytes, encoding: .isoLatin1)) + var tab = fileTab(query: latin1, url: url) + + FileTabBaseline.hydrate(&tab) + + #expect(tab.content.query == Self.japaneseScript) + #expect(tab.content.isFileDirty == false) + #expect(tab.content.sourceFileEncoding?.encoding == .shiftJIS) + } + + @Test("An older tab edited over the Latin-1 reading keeps that reading, so saving loses nothing") + func olderEditedTabKeepsTheLatin1Reading() throws { + let (url, bytes) = try makeShiftJISFile() + defer { try? FileManager.default.removeItem(at: url) } + let latin1 = try #require(String(data: bytes, encoding: .isoLatin1)) + var tab = fileTab(query: latin1 + "-- edited\n", url: url) + + FileTabBaseline.hydrate(&tab) + + #expect(tab.content.isFileDirty) + #expect(tab.content.savedFileContent == latin1) + #expect(tab.content.sourceFileEncoding?.encoding == .isoLatin1) + } + + @Test("A persisted tab carries its file's encoding, and a record without one still decodes") + func persistedTabCarriesTheEncoding() throws { + var tab = fileTab(query: "SELECT 1", url: URL(fileURLWithPath: "/tmp/report.sql")) + tab.content.sourceFileEncoding = FileTextEncoding(encoding: .shiftJIS) + let data = try JSONEncoder().encode(tab.toPersistedTab()) + let decoded = try JSONDecoder().decode(PersistedTab.self, from: data) + #expect(decoded.sourceFileEncoding?.encoding == .shiftJIS) + #expect(QueryTab(from: decoded, defaultPageSize: 100).content.sourceFileEncoding?.encoding == .shiftJIS) + + var json = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + json.removeValue(forKey: "sourceFileEncoding") + let older = try JSONDecoder().decode(PersistedTab.self, from: JSONSerialization.data(withJSONObject: json)) + #expect(older.sourceFileEncoding == nil) + } + + @Test("A recorded byte order mark the file no longer has falls back to reading the file as it is now") + func recordedMarkTheFileLost() throws { + let url = try makeFile(contents: "SELECT 1") + defer { try? FileManager.default.removeItem(at: url) } + var tab = fileTab(query: "SELECT 1", url: url) + tab.content.sourceFileEncoding = FileTextEncoding(encoding: .utf8, byteOrderMark: .utf8) + + FileTabBaseline.hydrate(&tab) + + #expect(tab.content.savedFileContent == "SELECT 1") + #expect(tab.content.isFileDirty == false) + } } diff --git a/TableProTests/Plugins/CSVImportEncodingFailureTests.swift b/TableProTests/Plugins/CSVImportEncodingFailureTests.swift new file mode 100644 index 0000000000..cd77f5809c --- /dev/null +++ b/TableProTests/Plugins/CSVImportEncodingFailureTests.swift @@ -0,0 +1,106 @@ +// +// CSVImportEncodingFailureTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class RecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? = "people" + + private(set) var insertedRows = 0 + private(set) var deletedAll = false + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { insertedRows += 1 } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { insertedRows += rows.count } + func deleteAllRowsFromTargetTable() async throws { deletedAll = true } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +private final class FileSource: PluginImportSource, @unchecked Sendable { + private let url: URL + + init(url: URL) { self.url = url } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func fileURL() -> URL { url } + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite("CSV import refuses text it cannot read", .serialized) +struct CSVImportEncodingFailureTests { + private func runImport(_ bytes: Data) async throws -> (Result, RecordingSink) { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("csv-import-\(UUID().uuidString).csv") + try bytes.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let plugin = CSVImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings = CSVImportOptions() + plugin.settings.deleteExistingRows = true + + let sink = RecordingSink() + do { + let result = try await plugin.performImport( + source: FileSource(url: url), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + return (.success(result), sink) + } catch { + return (.failure(error), sink) + } + } + + @Test("A Shift JIS file with a broken line is refused before any row is deleted or inserted") + func brokenShiftJISIsRefused() async throws { + var bytes = try #require("氏名,住所\n山田太郎,東京都港区\n鈴木花子,大阪市北区\n".data(using: .shiftJIS)) + bytes.append(contentsOf: [0xA0, 0x2C, 0x78, 0x0A]) + let (outcome, sink) = try await runImport(bytes) + guard case .failure(let error) = outcome else { + Issue.record("The import went ahead over a line it could not read") + return + } + #expect(error.localizedDescription.contains("Line 4")) + #expect(error.localizedDescription.contains("Shift JIS")) + #expect(!sink.deletedAll) + #expect(sink.insertedRows == 0) + } + + @Test("A UTF-8 file with an invalid byte is refused before any row is deleted or inserted") + func brokenUTF8IsRefused() async throws { + var bytes = Data("name,city\nJürgen,Köln\nZoë,Paris\n".utf8) + bytes.append(contentsOf: [0x43, 0x61, 0x66, 0xE9, 0x2C, 0x78, 0x0A]) + let (outcome, sink) = try await runImport(bytes) + guard case .failure = outcome else { + Issue.record("The import went ahead over a line it could not read") + return + } + #expect(!sink.deletedAll) + #expect(sink.insertedRows == 0) + } + + @Test("A clean Shift JIS file imports every row") + func cleanShiftJISImports() async throws { + let bytes = try #require("氏名,住所\n山田太郎,東京都港区\n鈴木花子,大阪市北区\n".data(using: .shiftJIS)) + let (outcome, sink) = try await runImport(bytes) + guard case .success = outcome else { + Issue.record("A readable Shift JIS file must import") + return + } + #expect(sink.insertedRows == 2) + } +} diff --git a/TableProTests/Plugins/CSVImportPluginTests.swift b/TableProTests/Plugins/CSVImportPluginTests.swift index e950f38d7d..ebf21e1592 100644 --- a/TableProTests/Plugins/CSVImportPluginTests.swift +++ b/TableProTests/Plugins/CSVImportPluginTests.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProTabularIO import Testing struct CSVImportPluginTests { @@ -32,14 +33,26 @@ struct CSVImportPluginTests { #expect(dialect.delimiter == 0x3B) } - @Test("Quote character and forced encoding are applied") - func testQuoteAndEncodingOverride() { + @Test("Quote character is applied, and the prepared text is always parsed as UTF-8") + func testQuoteOverrideParsesUTF8() { var options = CSVImportOptions() options.quoteCharacter = .singleQuote options.encoding = .isoLatin1 let dialect = CSVImportParsing.resolveDialect(in: data("a,b\n1,2\n"), options: options) #expect(dialect.quoteChar == 0x27) - #expect(dialect.encoding == .isoLatin1) + #expect(dialect.escapeChar == 0x27) + #expect(dialect.encoding == .utf8) + } + + @Test("A double quote inside a single-quoted field stays in the field") + func testDoubleQuoteInsideSingleQuotedField() { + var options = CSVImportOptions() + options.quoteCharacter = .singleQuote + options.hasHeaderRow = true + let detected = CSVImportParsing.detectFields(in: data("a,b\n'say \"hi\"',x\nnext,row\n"), options: options) + #expect(detected.map(\.name) == ["a", "b"]) + #expect(fields("a", detected)?.sampleValue == "say \"hi\"") + #expect(fields("b", detected)?.sampleValue == "x") } // MARK: - Column names @@ -206,4 +219,114 @@ struct CSVImportPluginTests { #expect(fields("n", result)?.inferredType == .integer) #expect(fields("n", result)?.sampleValue == "5") } + + // MARK: - File encodings + + private func writeFile(_ bytes: Data, fileExtension: String = "csv") throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("CSVImportPluginTests-\(UUID().uuidString).\(fileExtension)") + try bytes.write(to: url) + return url + } + + private func shiftJIS(_ text: String) throws -> Data { + try #require(text.data(using: .shiftJIS, allowLossyConversion: false)) + } + + private func fields(ofFile bytes: Data, options: CSVImportOptions = CSVImportOptions(), fileExtension: String = "csv") throws -> [PluginImportField] { + let url = try writeFile(bytes, fileExtension: fileExtension) + defer { try? FileManager.default.removeItem(at: url) } + let text = try CSVImportText.prefix(of: url, length: 1_048_576, encoding: options.encoding) + return CSVImportParsing.detectFields(in: text.data, options: options) + } + + @Test("A Shift JIS export keeps its commas when full-width dashes and ポ carry a pipe byte") + func testShiftJISCommaFileWithPipeTrailBytes() throws { + let bytes = try shiftJIS(""" + 顧客ID,氏名,住所,電話 + 1,山田太郎,東京都港区芝公園４－２－８,03－1234－5678 + 2,ポイント商事,大阪府大阪市北区梅田１－１,06－9876－5432 + 3,ソフト表示株式会社,愛知県名古屋市中区栄３－５,052－111－2222 + + """) + let detected = try fields(ofFile: bytes) + #expect(detected.map(\.name) == ["顧客ID", "氏名", "住所", "電話"]) + #expect(fields("住所", detected)?.sampleValue == "東京都港区芝公園４－２－８") + #expect(fields("氏名", detected)?.sampleValue == "山田太郎") + } + + @Test("A pipe-delimited Shift JIS file splits on real pipes only") + func testShiftJISPipeDelimitedFile() throws { + let bytes = try shiftJIS("名称|読み\nポイント交換|ぽいんとこうかん\nソフト表示|そふとひょうじ\n") + let detected = try fields(ofFile: bytes) + #expect(detected.map(\.name) == ["名称", "読み"]) + #expect(fields("名称", detected)?.sampleValue == "ポイント交換") + } + + @Test("A UTF-16 file with a byte order mark imports as text, not as split code units") + func testUTF16TabSeparatedFile() throws { + let text = "名前\t住所\n吉田\t三上\n本田\t日本\n" + let bytes = Data([0xFF, 0xFE]) + (try #require(text.data(using: .utf16LittleEndian))) + let detected = try fields(ofFile: bytes, fileExtension: "tsv") + #expect(detected.map(\.name) == ["名前", "住所"]) + #expect(fields("住所", detected)?.sampleValue == "三上") + } + + @Test("A chosen encoding wins over detection") + func testChosenEncodingIsHonored() throws { + var options = CSVImportOptions() + options.encoding = .isoLatin1 + let bytes = Data([0x6E, 0x61, 0x6D, 0x65, 0x0A, 0x43, 0x61, 0x66, 0xE9, 0x0A]) + let detected = try fields(ofFile: bytes, options: options) + #expect(fields("name", detected)?.sampleValue == "Caf\u{E9}") + } + + private func makeCopyDirectory() throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("CSVImportPluginTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + private func copies(in directory: URL) throws -> [String] { + try FileManager.default.contentsOfDirectory(atPath: directory.path) + } + + @Test("The whole-file import reads the same text as the preview and removes its UTF-8 copy") + func testContentsTranscodeAndCleanUp() throws { + let url = try writeFile(try shiftJIS("顧客ID,氏名\n1,髙橋\n")) + let directory = try makeCopyDirectory() + defer { + try? FileManager.default.removeItem(at: url) + try? FileManager.default.removeItem(at: directory) + } + let text = try CSVImportText.contents(of: url, encoding: .auto, copyingInto: directory) + #expect(text.encoding == .shiftJIS) + #expect(String(data: text.data, encoding: .utf8) == "顧客ID,氏名\n1,髙橋\n") + #expect(try copies(in: directory).count == 1) + text.removeTemporaryFile() + #expect(try copies(in: directory).isEmpty) + } + + @Test("Stopping the import stops the transcode and leaves no UTF-8 copy") + func testTranscodeStopsWhenCancelled() throws { + let url = try writeFile(try shiftJIS(String(repeating: "1,髙橋\n", count: 400_000))) + let directory = try makeCopyDirectory() + defer { + try? FileManager.default.removeItem(at: url) + try? FileManager.default.removeItem(at: directory) + } + #expect(throws: TabularCancellation.self) { + try CSVImportText.contents(of: url, encoding: .auto, copyingInto: directory, isCancelled: { true }) + } + #expect(try copies(in: directory).isEmpty) + } + + @Test("Every encoding option names a real encoding") + func testEncodingOptionsMapToTabularEncodings() { + for option in CSVImportOptions.TextEncoding.allCases where option != .auto { + #expect(option.tabularEncoding != nil, "\(option)") + } + #expect(CSVImportOptions.TextEncoding.auto.tabularEncoding == nil) + } } diff --git a/TableProTests/Plugins/JSONImportPluginTests.swift b/TableProTests/Plugins/JSONImportPluginTests.swift index 1d61e73eb1..d261301f32 100644 --- a/TableProTests/Plugins/JSONImportPluginTests.swift +++ b/TableProTests/Plugins/JSONImportPluginTests.swift @@ -183,4 +183,33 @@ struct JSONImportPluginTests { #expect(fields.first { $0.name == "name" }?.inferredType == .text) #expect(fields.first { $0.name == "id" }?.sampleValue == "1") } + + // MARK: - JSON Lines sample + + @Test("A JSON Lines sample cut inside a Japanese character still lists its fields") + func testSampleCutInsideAMultiByteCharacter() throws { + let line = Data("{\"名前\":\"山田太郎\",\"住所\":\"東京都港区\"}\n".utf8) + var body = Data() + while body.count <= JSONImportParsing.sampleLength { + body.append(line) + } + var file = body + while String(data: file.prefix(JSONImportParsing.sampleLength), encoding: .utf8) != nil { + file.insert(0x0A, at: 0) + } + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("JSONImportPluginTests-\(UUID().uuidString).jsonl") + try file.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let rows = try JSONImportParsing.sampleRawRows(at: url, targetTable: nil, limit: 5) + #expect(rows.count == 5) + #expect(rows.first?["名前"] as? String == "山田太郎") + } + + @Test("A sample of whole characters decodes unchanged") + func testSampleOfWholeCharacters() { + #expect(JSONImportParsing.utf8Text(ofSample: Data("{\"a\":\"日本\"}".utf8)) == "{\"a\":\"日本\"}") + #expect(JSONImportParsing.utf8Text(ofSample: Data([0x7B, 0xE6, 0x97])) == "{") + } } diff --git a/TableProUITests/DataFileWindowUITests.swift b/TableProUITests/DataFileWindowUITests.swift index e9934620f6..266ea6df50 100644 --- a/TableProUITests/DataFileWindowUITests.swift +++ b/TableProUITests/DataFileWindowUITests.swift @@ -119,6 +119,33 @@ final class DataFileWindowUITests: UITestCase { XCTAssertTrue(waitForRowCount("5 rows", in: window), "Choosing a sheet must show that sheet's rows") } + func testAShiftJISExportOpensWithItsJapaneseText() throws { + let export = "商品コード,商品名,在庫管理区分\n10001,ソフトウェア保守,在庫管理する\n10002,ポイント交換,在庫管理しない\n10003,髙橋商店の注文,在庫管理する\n" + let bytes = try XCTUnwrap(export.data(using: .shiftJIS, allowLossyConversion: false)) + let app = try launchWithDataFile(named: "shift-jis.csv", contents: bytes) + let (window, grid) = try readyWindow(of: app) + XCTAssertTrue(waitForRowCount("3 rows", in: window), "The export must open with its three rows") + let format = window.descendants(matching: .any)["data-file-format"].firstMatch + XCTAssertTrue(format.waitToExist(timeout: 15), "The status bar must describe the file format") + XCTAssertTrue( + waitForPredicate(timeout: 15) { "\(format.label) \(format.value ?? "")".contains("Shift JIS") }, + "The status bar must name the detected encoding, not \(format.label) \(String(describing: format.value))" + ) + + cellPoint(in: grid, row: 1).click() + app.typeKey("i", modifierFlags: [.command, .option]) + let header = window.descendants(matching: .any)["data-file-row-details-header"].firstMatch + XCTAssertTrue(header.waitToExist(timeout: 10), "Row Details must open beside the grid") + let name = window.descendants(matching: .any) + .matching(NSPredicate(format: "value == %@ OR label == %@", "ポイント交換", "ポイント交換")) + XCTAssertTrue( + waitForPredicate(timeout: 10) { + name.allElementsBoundByIndex.contains { $0.frame.minX >= header.frame.minX - 1 } + }, + "Row Details must show the Japanese value, not Windows-1252 mojibake" + ) + } + // MARK: - Helpers /// Two sheets, People (3 rows) and Cities (5 rows), with inline strings and a header row each. diff --git a/docs/external-api/ios-shortcuts.mdx b/docs/external-api/ios-shortcuts.mdx index 3c559e3b24..f2b37d5c24 100644 --- a/docs/external-api/ios-shortcuts.mdx +++ b/docs/external-api/ios-shortcuts.mdx @@ -77,7 +77,9 @@ digits, while `"18446744073709551615"` keeps every one. ### CSV -The first row is the header and names the columns. Quoting follows the usual rules. +The first row is the header and names the columns. Quoting follows the usual rules. A file can be +UTF-8, UTF-16, Shift JIS, EUC-JP, GB 18030, Big5, EUC-KR or Windows-1252; the encoding is read +from the file. ```text title,url @@ -100,6 +102,7 @@ Nothing is inserted in any of these cases. | *"…has no column named …"* | A key in the data matches no column. | | *"The data has no values to insert into …"* | Nothing in the data maps to a column. | | *"No data was provided to add."* | Both the text field and the file were empty. | +| *"Line … of the file is not valid … text."* | That line holds bytes the file's encoding cannot read. | | *"Too many rows. Add up to 10000 rows at a time."* | Over the cap. | | *"Add Row to Table expects one row. Use Add Rows to Table for multiple rows."* | An array reached the single-row action. | | *"…check the server's SSH host key first."* or *"…Open this connection in TablePro to check it."* | The SSH server is new, or its host key changed. A Shortcut has no screen to show the question on. | diff --git a/docs/features/data-files.mdx b/docs/features/data-files.mdx index 3c5d77c0a3..dc1352b11e 100644 --- a/docs/features/data-files.mdx +++ b/docs/features/data-files.mdx @@ -29,15 +29,15 @@ The window works on a private copy of the file taken when it opens, so another a | Setting | How it is chosen | |---|---| -| Encoding | A byte order mark picks UTF-8, UTF-16 LE or UTF-16 BE. Without one, the first 256 KB decide: valid UTF-8 reads as UTF-8, anything else as Windows-1252 | +| Encoding | A byte order mark picks UTF-8, UTF-16 LE or UTF-16 BE. Without one, 64 KB starting at the first line with a non-ASCII byte decide: valid UTF-8 reads as UTF-8, and anything else is matched against Shift JIS, EUC-JP, GB 18030, Big5, EUC-KR and Windows-1252. UTF-16 without a mark is recognized by its zero bytes | | Delimiter | Comma, tab, semicolon or pipe, whichever appears most often outside quotes in the first 64 KB. `.tsv` always uses tab and `.psv` always uses pipe | | Line ending | CRLF, LF or CR, whichever comes first outside quotes | | Header row | Row one is the header when at least half of its cells are text rather than numbers | | Column types | Integer, decimal, boolean, date or text, from the first 200 non-blank values in each column | -Two columns with the same header, or an empty one, get unique names such as `amount (2)` and `Column 3`. A row with more fields than the first row widens the table, and the status bar counts how many rows differ. +Two columns with the same header, or an empty one, get unique names such as `amount (2)` and `Column 3`. A row with more fields than the first row widens the table, and the status bar counts how many rows differ. A byte the encoding cannot read shows as `�`. In a Shift JIS, EUC-JP, GB 18030, Big5, EUC-KR or UTF-16 file, the status bar also counts the lines that hold one. -When a guess is wrong, choose **Edit > Data > File Properties…** to set the delimiter, quote, escape character, encoding, line ending and header row by hand, then click **Reload**. To flip only the header, choose **Edit > Data > Switch First Row Between Header and Data** (`Cmd+Shift+H`). **Change Type** on a column header overrides an inferred type, which changes how the column sorts, filters and appears in statistics, never what is written to disk. +When a guess is wrong, choose **Edit > Data > File Properties…** to set the delimiter, quote, escape character, encoding, line ending and header row by hand, then click **Reload**. The window keeps those settings when it reloads the file after another app changes it. To flip only the header, choose **Edit > Data > Switch First Row Between Header and Data** (`Cmd+Shift+H`). **Change Type** on a column header overrides an inferred type, which changes how the column sorts, filters and appears in statistics, never what is written to disk. ## Filter, search and sort @@ -127,7 +127,7 @@ A workbook is read-only. To keep a sheet, choose **File > Save As** and pick CSV ## Save and export -`Cmd+S` writes the file back in its own format. Rows you did not touch are copied byte for byte, so a large file with a handful of edits saves quickly and differs from the original only where you changed it. If a value holds a character the file's encoding cannot store, the save stops and names the row and column. Choose **File > Save As** and set **Text Encoding** to UTF-8 to keep it; later saves keep the new encoding. +`Cmd+S` writes the file back in its own format. Rows you did not touch are copied byte for byte, so a large file with a handful of edits saves quickly and differs from the original only where you changed it. If a value holds a character the file's encoding cannot store, the save stops, names the row as the grid numbers it and the column by name, and selects that row. In a Shift JIS or EUC-JP file, `¥` saves as the byte Japanese Windows shows as `¥`, which reads back as `\`. Choose **File > Save As** and set **Text Encoding** to UTF-8 to keep it; later saves keep the new encoding. **File > Save As** (`Cmd+Shift+S`) writes a delimited file as CSV, TSV or pipe-separated text, and a JSON file as JSON, JSON Lines, CSV or TSV. diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 0817ca5eb0..4f4efa886a 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -287,7 +287,7 @@ The extension picks the format, so one command covers all of them. A file no for | Option | What it does | Default | |--------|-------------|---------| | On error | Stop and Rollback, Stop and Commit, or Skip and Continue | Stop and Rollback | -| Encoding | UTF-8, UTF-16, UTF-16 LE, UTF-16 BE, Latin-1, Windows-1252, or ASCII | UTF-8 | +| Encoding | UTF-8, UTF-16, UTF-16 LE, UTF-16 BE, Latin-1, Windows-1252, ASCII, Shift JIS, EUC-JP, GB 18030, Big5, or EUC-KR | UTF-8 | | Wrap in transaction | Runs every statement inside one transaction. Dimmed in Skip and Continue | Yes | | Disable foreign key checks | Suspends constraint checks for the import | Yes | @@ -299,6 +299,8 @@ The extension picks the format, so one command covers all of them. A file no for **UTF-16** reads the byte order from the mark at the start of the file and falls back to big-endian, which is what a file with no mark means. Pick **UTF-16 LE** or **UTF-16 BE** for a file that has no mark and is not big-endian. Latin-1 and Windows-1252 differ over the bytes `0x80` to `0x9F`: a dump written by MySQL keeps its curly quotes, en dashes and euro sign there, so Windows-1252 is the one to pick for it. +When the preview cannot read the file in the chosen encoding, the dialog switches to the encoding the dump declares in its `SET NAMES` or `SET client_encoding`, or else to the encoding the text matches. It picks Shift JIS, GB 18030 or Big5 only when the dump declares it, because a byte in those encodings can hide a quote's escaping backslash; choose one by hand for a script that declares nothing. A switch the dialog made is not remembered for the next import. A dump read in any encoding but UTF-8 or ASCII reaches the server as UTF-8, so its `SET NAMES cp932` or `SET client_encoding = 'SJIS'` does not run as written: MySQL and MariaDB keep the connection's own character set, and PostgreSQL gets `SET client_encoding = 'UTF8'`. On MySQL and MariaDB, a `_binary` value in a Latin-1 or Windows-1252 dump is sent as the bytes it held. Any other non-UTF-8 dump stops at a `_binary` value holding non-ASCII bytes, since those cannot be read back exactly; export it with `--hex-blob` instead. + On SQL Server a file that holds a `GO` line runs a batch at a time, cut at its `GO` lines the way `sqlcmd` cuts it, so every count and failure here is a batch rather than a statement. A file with no `GO` line runs a statement at a time. See [Scripts and batches](/databases/mssql#scripts-and-batches). Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. The alert shows the first 10,000 characters of a longer statement and the first five errors of a batch that raised more, and **Copy Details** copies all of them. @@ -350,13 +352,13 @@ CSV and TSV open the same sheet as JSON, with parsing options in front of the ma |--------|-------------|---------| | Delimiter | Comma, semicolon, tab, or pipe | Auto-detect | | Quote character | Double or single quote | Double quote (`"`) | -| Encoding | UTF-8, ISO Latin 1, or Windows-1252 | Auto-detect | +| Encoding | UTF-8, UTF-16 LE, UTF-16 BE, ISO Latin 1, Windows-1252, Shift JIS, EUC-JP, GB 18030, Big5, or EUC-KR | Auto-detect | | First row is a header | Row 1 becomes the column names; off imports it as data | Yes | | Trim leading and trailing spaces | Trims each field before import | No | | Treat empty values as NULL | Inserts NULL instead of empty text | Yes | | NULL text | One more value to import as NULL, `\N` for instance | None | -Quoted fields keep embedded commas and newlines (RFC 4180), and a doubled quote (`""`) decodes to one. +Quoted fields keep embedded commas and newlines (RFC 4180), and a doubled quote (`""`) decodes to one. A line the encoding cannot read stops the import before any row is deleted or inserted, and the error names the line. ### Row imports diff --git a/docs/features/sql-files.mdx b/docs/features/sql-files.mdx index 45b07580da..eb323f838c 100644 --- a/docs/features/sql-files.mdx +++ b/docs/features/sql-files.mdx @@ -11,7 +11,7 @@ Three ways in: double-click in Finder, **File > Open File…** (`Cmd+O`), or dra Each file opens in a new tab. Opening one that is already open focuses its existing tab instead of making a second, and files opened before any connection exists wait in a queue until you connect. -The encoding comes from the file: its byte order mark first, then the encoding a Mac app recorded for it when saving, then UTF-8. Bytes that fit none of those open as ISO Latin-1. +The encoding comes from the file: its byte order mark first, then the encoding a Mac app recorded for it when saving, then UTF-8, then UTF-16 or an East Asian encoding (Shift JIS, EUC-JP, GB 18030, Big5 or EUC-KR) matched from the text. Bytes that fit none of those open as ISO Latin-1. The title bar carries the filename, and the close button gets the standard macOS unsaved-changes dot. @@ -29,7 +29,7 @@ Closing a query tab keeps its SQL either way. Reopen it with `Cmd+Shift+T` or ** A save writes the file in the encoding it was opened in. A byte order mark stays, and UTF-16 and UTF-32 keep their byte order. A file known only by the encoding a Mac app recorded for it keeps that record. A file recorded as ASCII saves as UTF-8. -If the text holds a character the encoding cannot store, such as an emoji in an ISO Latin-1 file, the save stops. An alert names the file and its encoding, the file on disk is left as it was, and the tab stays open with your text. This holds for `Cmd+S`, for **Save** when closing a tab or a window, and for closing several tabs at once. +In a Shift JIS or EUC-JP file, `¥` saves as the byte Japanese Windows shows as `¥`, which reads back as `\`. If the text holds a character the encoding cannot store, such as an emoji in an ISO Latin-1 file, the save stops. An alert names the file and its encoding, the file on disk is left as it was, and the tab stays open with your text. This holds for `Cmd+S`, for **Save** when closing a tab or a window, and for closing several tabs at once. Save As always writes UTF-8, which is how to convert a file. The tab then points at the new file and keeps saving it as UTF-8. diff --git a/project.yml b/project.yml index 50aa44a369..7871c30807 100644 --- a/project.yml +++ b/project.yml @@ -440,7 +440,10 @@ targets: - Plugins/CSVExportPlugin/CSVExportOptionsView.swift - Plugins/CSVExportPlugin/CSVExportPlugin.swift - Plugins/CSVImportPlugin/CSVImportOptions.swift + - Plugins/CSVImportPlugin/CSVImportOptionsView.swift - Plugins/CSVImportPlugin/CSVImportParsing.swift + - Plugins/CSVImportPlugin/CSVImportPlugin.swift + - Plugins/CSVImportPlugin/CSVImportText.swift - Plugins/ClickHouseDriverPlugin/ClickHouseCapabilities.swift - Plugins/ClickHouseDriverPlugin/ClickHouseCredentials.swift - Plugins/ClickHouseDriverPlugin/ClickHouseGeneratedColumnClassification.swift @@ -1150,6 +1153,9 @@ targets: templateAttributes: folder: CSVImportPlugin principalClass: CSVImportPlugin + dependencies: + - package: TableProCore + product: TableProTabularIO settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.CSVImportPlugin diff --git a/scripts/benchmark/data-files/main.swift b/scripts/benchmark/data-files/main.swift index e11c9f04c4..0ff7191815 100644 --- a/scripts/benchmark/data-files/main.swift +++ b/scripts/benchmark/data-files/main.swift @@ -11,11 +11,10 @@ enum DataFilesBenchmark { } let url = URL(fileURLWithPath: CommandLine.arguments[1]) let bytes = try Data(contentsOf: url, options: .alwaysMapped) + let sniff = TabularEncodingDetector.sniff(bytes) let dialect = bytes.withUnsafeBytes { raw -> DelimitedDialect in - let buffer = raw.bindMemory(to: UInt8.self) - let sniff = DelimitedDialectDetector.sniffEncoding(buffer) - return DelimitedDialectDetector.detect( - buffer, + DelimitedDialectDetector.detect( + raw.bindMemory(to: UInt8.self), contentStart: sniff.byteOrderMarkLength, encoding: sniff.encoding, hasByteOrderMark: sniff.hasByteOrderMark,