From af870b3a485a7ecf1f4a467f7994a09cc2c4dc86 Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Sat, 17 Oct 2026 22:41:42 +0200 Subject: [PATCH 1/2] replaces the hard-coded dialect list in the tokenizer with a `supports_hash_comment` dialect method --- src/dialect/bigquery.rs | 4 ++++ src/dialect/clickhouse.rs | 5 +++++ src/dialect/hive.rs | 4 ++++ src/dialect/mod.rs | 6 ++++++ src/dialect/mysql.rs | 4 ++++ src/dialect/snowflake.rs | 4 ++++ src/tokenizer.rs | 10 +++------- tests/sqlparser_common.rs | 21 +++++++++++++-------- 8 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/dialect/bigquery.rs b/src/dialect/bigquery.rs index d34be9a40c..4dff5ea945 100644 --- a/src/dialect/bigquery.rs +++ b/src/dialect/bigquery.rs @@ -72,6 +72,10 @@ impl Dialect for BigQueryDialect { Some('`') } + fn supports_hash_comment(&self) -> bool { + true + } + fn supports_projection_trailing_commas(&self) -> bool { true } diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index d256da28ea..5865147d79 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -135,6 +135,11 @@ impl Dialect for ClickHouseDialect { true } + /// See + fn supports_hash_comment(&self) -> bool { + true + } + /// See fn supports_optimize_table(&self) -> bool { true diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index 1a26f56ba2..1a7d6de9e3 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -27,6 +27,10 @@ impl Dialect for HiveDialect { (ch == '"') || (ch == '`') } + fn supports_hash_comment(&self) -> bool { + true + } + /// See fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('`') diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index d3778994dc..d2615b89f7 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1281,6 +1281,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports `#` as the start of a single line comment, + /// e.g. `SELECT 1 # comment` + fn supports_hash_comment(&self) -> bool { + false + } + /// Returns true if the dialect supports optimizer hints in multiline comments /// e.g. `/*!50110 KEY_BLOCK_SIZE = 1024*/` fn supports_multiline_comment_hints(&self) -> bool { diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index 97bdfe3051..74cab00fda 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -163,6 +163,10 @@ impl Dialect for MySqlDialect { true } + fn supports_hash_comment(&self) -> bool { + true + } + fn supports_match_against(&self) -> bool { true } diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index a25f286313..f1f9a3d9af 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -140,6 +140,10 @@ impl Dialect for SnowflakeDialect { ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_' } + fn supports_hash_comment(&self) -> bool { + true + } + /// See fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('"') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 423bbea310..2355b18463 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -40,16 +40,13 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +use crate::ast::{DollarQuotedString, QuoteDelimitedString}; use crate::dialect::Dialect; use crate::dialect::{ BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, SnowflakeDialect, }; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; -use crate::{ - ast::{DollarQuotedString, QuoteDelimitedString}, - dialect::HiveDialect, -}; /// SQL Token enumeration #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -1736,9 +1733,8 @@ impl<'a> Tokenizer<'a> { } '{' => self.consume_and_return(chars, Token::LBrace), '}' => self.consume_and_return(chars, Token::RBrace), - '#' if dialect_of!(self is SnowflakeDialect | BigQueryDialect | MySqlDialect | HiveDialect) => - { - chars.next(); // consume the '#', starting a snowflake single-line comment + '#' if self.dialect.supports_hash_comment() => { + chars.next(); // consume the '#', starting a single-line comment let comment = self.tokenize_single_line_comment(chars); Ok(Some(Token::Whitespace(Whitespace::SingleLineComment { prefix: "#".to_owned(), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 8b7a51e1fe..f7c3315f26 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -13258,22 +13258,27 @@ fn test_release_savepoint() { #[test] fn test_comment_hash_syntax() { - let dialects = TestedDialects::new(vec![ - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(MySqlDialect {}), - Box::new(HiveDialect {}), - ]); let sql = r#" # comment SELECT a, b, c # , d, e FROM T ####### comment ################# - WHERE true + WHERE true # it's a comment # comment "#; let canonical = "SELECT a, b, c FROM T WHERE true"; - dialects.verified_only_select_with_canonical(sql, canonical); + all_dialects_where(|d| d.supports_hash_comment()) + .verified_only_select_with_canonical(sql, canonical); + + // Elsewhere `#` is an operator, so the apostrophe starts an unterminated string. + assert_eq!( + all_dialects_where(|d| !d.supports_hash_comment()) + .parse_sql_statements("SELECT 1 # it's a comment") + .unwrap_err(), + ParserError::TokenizerError( + "Unterminated string literal at Line: 1, Column: 14".to_string() + ) + ); } #[test] From 34a56487f7220a0a217f33dbbc03401805a3036f Mon Sep 17 00:00:00 2001 From: s5dsn-eqee Date: Sun, 18 Oct 2026 14:40:06 +0200 Subject: [PATCH 2/2] intro requires_hash_comment_space --- fuzz/fuzz_seeds/clickhouse_hash_comment.sql | 48 ++++++++++++++++++++ src/dialect/clickhouse.rs | 5 +++ src/dialect/mod.rs | 6 +++ src/tokenizer.rs | 9 +++- tests/sqlparser_common.rs | 49 +++++++++++++++++++-- 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 fuzz/fuzz_seeds/clickhouse_hash_comment.sql diff --git a/fuzz/fuzz_seeds/clickhouse_hash_comment.sql b/fuzz/fuzz_seeds/clickhouse_hash_comment.sql new file mode 100644 index 0000000000..d9a269e321 --- /dev/null +++ b/fuzz/fuzz_seeds/clickhouse_hash_comment.sql @@ -0,0 +1,48 @@ +#!/usr/bin/clickhouse-local --queries-file +# leading line comment +#! shebang style comment +SELECT 1 # trailing comment +; +SELECT a, b # , c +FROM t # table +WHERE x = 1 # it's a comment with an apostrophe +; +SELECT # comment right after SELECT + a, + #! comment between select items + b +FROM t; +SELECT 1 #!no space after bang, the semicolon is part of the comment; +; +SELECT 1 # several spaces then comment +; +SELECT '# not a comment', "# not a comment", `# not a comment` FROM t # real comment +; +SELECT a FROM t WHERE s = 'a # b' # comment containing 'quotes' and "double quotes" +; +SELECT a FROM t -- dash comment +# hash comment +/* block comment */ +#! shebang comment +WHERE a = 1; +SELECT /* # not a comment */ 1 # comment with /* block */ inside +; +SELECT 1 # comment with -- inside +; +SELECT 1 # comment with # another hash # and more +; +SELECT * FROM t PREWHERE x GLOBAL IN (1, 2) # after prewhere +SETTINGS max_threads = 1 # after settings +; +INSERT INTO t2 SELECT * FROM t # comment in insert +; +CREATE TABLE t (a Int32, # comment after column + b String # comment after last column +) ENGINE = MergeTree ORDER BY a # comment after order by +; +WITH c AS (SELECT 1 AS v) # comment after cte +SELECT * FROM c; +SELECT arrayMap(x -> x + 1, # comment inside call + [1, 2]) # comment after call +; +SELECT 1 # comment at end of file without newline \ No newline at end of file diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index 5865147d79..3f5e741dec 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -140,6 +140,11 @@ impl Dialect for ClickHouseDialect { true } + /// See + fn requires_hash_comment_space(&self) -> bool { + true + } + /// See fn supports_optimize_table(&self) -> bool { true diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index c4a06413d8..2bf1f6d6ef 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1297,6 +1297,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if `#` starts a single line comment only when followed by an ASCII + /// space or `!`, so `# comment` and `#! comment` are comments but `#comment` is not. + fn requires_hash_comment_space(&self) -> bool { + false + } + /// Returns true if the dialect supports optimizer hints in multiline comments /// e.g. `/*!50110 KEY_BLOCK_SIZE = 1024*/` fn supports_multiline_comment_hints(&self) -> bool { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 67d6e470a2..4e76be8417 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1733,7 +1733,14 @@ impl<'a> Tokenizer<'a> { } '{' => self.consume_and_return(chars, Token::LBrace), '}' => self.consume_and_return(chars, Token::RBrace), - '#' if self.dialect.supports_hash_comment() => { + '#' if self.dialect.supports_hash_comment() + && (!self.dialect.requires_hash_comment_space() + || chars + .peekable + .clone() + .nth(1) + .is_some_and(|c| c == ' ' || c == '!')) => + { chars.next(); // consume the '#', starting a single-line comment let comment = self.tokenize_single_line_comment(chars); Ok(Some(Token::Whitespace(Whitespace::SingleLineComment { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 1237a5447d..97a484ef7d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -39,8 +39,7 @@ use sqlparser::dialect::{ }; use sqlparser::keywords::{Keyword, ALL_KEYWORDS}; use sqlparser::parser::{Parser, ParserError, ParserOptions}; -use sqlparser::tokenizer::Tokenizer; -use sqlparser::tokenizer::{Location, Span}; +use sqlparser::tokenizer::{Location, Span, Token, Tokenizer, Whitespace}; use test_utils::{ all_dialects, all_dialects_where, all_dialects_with_options, alter_table_op, assert_eq_vec, call, expr_from_projection, join, number, only, table, table_alias, table_from_name, @@ -13262,7 +13261,7 @@ fn test_comment_hash_syntax() { # comment "#; let canonical = "SELECT a, b, c FROM T WHERE true"; - all_dialects_where(|d| d.supports_hash_comment()) + all_dialects_where(|d| d.supports_hash_comment() && !d.requires_hash_comment_space()) .verified_only_select_with_canonical(sql, canonical); // Elsewhere `#` is an operator, so the apostrophe starts an unterminated string. @@ -13276,6 +13275,50 @@ fn test_comment_hash_syntax() { ); } +#[test] +fn test_comment_hash_requires_space() { + let dialects = all_dialects_where(|d| d.requires_hash_comment_space()); + + let sql = r#" + # comment + #! shebang comment + SELECT a, b, c # , d, e + FROM T + WHERE true #! it's a comment + "#; + let canonical = "SELECT a, b, c FROM T WHERE true"; + dialects.verified_only_select_with_canonical(sql, canonical); + + // Without a space or `!` after it, `#` is an operator token, which the parser rejects. + for sql in [ + "SELECT 1 #comment", + "SELECT 1 ####### comment", + "SELECT 1 #", + "SELECT 1 #\tcomment", + ] { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("No infix parser for token Sharp".to_string()), + "{sql}" + ); + } + dialects.tokenizes_to( + "SELECT 1 ### comment", + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::Number("1".to_string(), false), + Token::Whitespace(Whitespace::Space), + Token::Sharp, + Token::Sharp, + Token::Whitespace(Whitespace::SingleLineComment { + prefix: "#".to_string(), + comment: " comment".to_string(), + }), + ], + ); +} + #[test] fn test_parse_inline_comment() { let sql =