From d307f57205b146f7dc0ee5edcd101ca51dc488ef Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Sun, 9 Aug 2026 07:40:14 +0200 Subject: [PATCH] fix(pg-protocol): decode binary result fields as raw bytes instead of utf-8 --- packages/pg-protocol/src/parser.ts | 20 ++++++-- .../gh-issues/binary-high-bytes-tests.js | 51 +++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 packages/pg/test/integration/gh-issues/binary-high-bytes-tests.js diff --git a/packages/pg-protocol/src/parser.ts b/packages/pg-protocol/src/parser.ts index df48ca4a1..1d18add4e 100644 --- a/packages/pg-protocol/src/parser.ts +++ b/packages/pg-protocol/src/parser.ts @@ -83,6 +83,8 @@ export class Parser { private bufferOffset: number = 0 private reader = new BufferReader() private mode: Mode + // Column formats from the most recent RowDescription, or null before any has been seen. + private rowDescriptionFormats: Mode[] | null = null constructor(opts?: StreamOptions) { if (opts?.mode === 'binary') { @@ -188,7 +190,7 @@ export class Parser { message = emptyQuery break case MessageCodes.DataRow: - message = parseDataRowMessage(reader) + message = parseDataRowMessage(reader, this.rowDescriptionFormats) break case MessageCodes.CommandComplete: message = parseCommandCompleteMessage(reader) @@ -214,9 +216,14 @@ export class Parser { case MessageCodes.NoticeMessage: message = parseErrorMessage(reader, 'notice') break - case MessageCodes.RowDescriptionMessage: - message = parseRowDescriptionMessage(reader) + case MessageCodes.RowDescriptionMessage: { + const rowDescription = parseRowDescriptionMessage(reader) + // Remember the column formats: the DataRows that follow carry no format information of + // their own, and message arrive in wire order, so the most recent RowDescription applies. + this.rowDescriptionFormats = rowDescription.fields.map((field) => field.format) + message = rowDescription break + } case MessageCodes.ParameterDescriptionMessage: message = parseParameterDescriptionMessage(reader) break @@ -306,13 +313,16 @@ const parseParameterDescriptionMessage = (reader: BufferReader) => { return message } -const parseDataRowMessage = (reader: BufferReader) => { +// `formats` comes from the RowDescription that precedes these DataRows. Fields the server sent in +// binary format are handed back as raw bytes; decoding them as utf-8 would silently corrupt any +// value containing a byte >= 0x80. Text fields keep their existing string representation. +const parseDataRowMessage = (reader: BufferReader, formats: Mode[] | null) => { const fieldCount = reader.int16() const fields: any[] = new Array(fieldCount) for (let i = 0; i < fieldCount; i++) { const len = reader.int32() // a -1 for length means the value of the field is null - fields[i] = len === -1 ? null : reader.string(len) + fields[i] = len === -1 ? null : formats?.[i] === 'binary' ? reader.bytes(len) : reader.string(len) } return new DataRowMessage(LATEINIT_LENGTH, fields) } diff --git a/packages/pg/test/integration/gh-issues/binary-high-bytes-tests.js b/packages/pg/test/integration/gh-issues/binary-high-bytes-tests.js new file mode 100644 index 000000000..9b9ef8941 --- /dev/null +++ b/packages/pg/test/integration/gh-issues/binary-high-bytes-tests.js @@ -0,0 +1,51 @@ +'use strict' +const helper = require('../test-helper') +const assert = require('assert') + +const suite = new helper.Suite() + +// Binary results used to be decoded as utf-8 by pg-protocol's BufferReader and then re-encoded by +// Result#parseRow with Buffer.from(). Any byte >= 0x80 does not survive that round trip: it is not +// valid utf-8 on its own, so it became U+FFFD (0xEF 0xBF 0xBD) and the original value was lost. +// +// The corruption started exactly at 0x80, which is why it went unnoticed: every value used by the +// existing binary tests happens to be built from bytes below that threshold. +// +// SELECT 127::int -> 127 (0x0000007F, all bytes < 0x80) +// SELECT 128::int -> 239 (0x00000080, corrupted) +// SELECT 200::int -> 239 (0x000000C8, corrupted) +// SELECT -1::int -> -272646673 (0xFFFFFFFF, corrupted) +suite.test('binary results survive bytes >= 0x80', async () => { + const client = new helper.pg.Client() + await client.connect() + + for (const value of [0, 1, 127, 128, 200, 255, 256, 65535, 2147483647, -1, -128, -2147483648]) { + const { rows } = await client.query({ text: 'SELECT $1::int AS a', values: [value], binary: true }) + assert.strictEqual(rows[0].a, value, `binary int ${value} round tripped as ${rows[0].a}`) + } + + await client.end() +}) + +suite.test('binary results preserve multi-byte text', async () => { + const client = new helper.pg.Client() + await client.connect() + + for (const value of ['wat', 'ciào €', '日本語', '🐘']) { + const { rows } = await client.query({ text: 'SELECT $1::text AS a', values: [value], binary: true }) + assert.strictEqual(rows[0].a, value) + } + + await client.end() +}) + +// Text mode is the default and must be untouched by the format-aware decoding. +suite.test('text mode is unaffected', async () => { + const client = new helper.pg.Client() + await client.connect() + + const { rows } = await client.query("SELECT 200::int AS a, 'ciào €'::text AS b, true AS c, NULL::int AS d") + assert.deepStrictEqual(rows[0], { a: 200, b: 'ciào €', c: true, d: null }) + + await client.end() +})