diff --git a/fuzz/fuzz_seeds/clickhouse_hash_comment.sql b/fuzz/fuzz_seeds/clickhouse_hash_comment.sql new file mode 100644 index 0000000000..d9a269e321 --- /dev/null +++ b/fuzz/fuzz_seeds/clickhouse_hash_comment.sql @@ -0,0 +1,48 @@ +#!/usr/bin/clickhouse-local --queries-file +# leading line comment +#! shebang style comment +SELECT 1 # trailing comment +; +SELECT a, b # , c +FROM t # table +WHERE x = 1 # it's a comment with an apostrophe +; +SELECT # comment right after SELECT + a, + #! comment between select items + b +FROM t; +SELECT 1 #!no space after bang, the semicolon is part of the comment; +; +SELECT 1 # several spaces then comment +; +SELECT '# not a comment', "# not a comment", `# not a comment` FROM t # real comment +; +SELECT a FROM t WHERE s = 'a # b' # comment containing 'quotes' and "double quotes" +; +SELECT a FROM t -- dash comment +# hash comment +/* block comment */ +#! shebang comment +WHERE a = 1; +SELECT /* # not a comment */ 1 # comment with /* block */ inside +; +SELECT 1 # comment with -- inside +; +SELECT 1 # comment with # another hash # and more +; +SELECT * FROM t PREWHERE x GLOBAL IN (1, 2) # after prewhere +SETTINGS max_threads = 1 # after settings +; +INSERT INTO t2 SELECT * FROM t # comment in insert +; +CREATE TABLE t (a Int32, # comment after column + b String # comment after last column +) ENGINE = MergeTree ORDER BY a # comment after order by +; +WITH c AS (SELECT 1 AS v) # comment after cte +SELECT * FROM c; +SELECT arrayMap(x -> x + 1, # comment inside call + [1, 2]) # comment after call +; +SELECT 1 # comment at end of file without newline \ No newline at end of file diff --git a/src/dialect/bigquery.rs b/src/dialect/bigquery.rs index d34be9a40c..4dff5ea945 100644 --- a/src/dialect/bigquery.rs +++ b/src/dialect/bigquery.rs @@ -72,6 +72,10 @@ impl Dialect for BigQueryDialect { Some('`') } + fn supports_hash_comment(&self) -> bool { + true + } + fn supports_projection_trailing_commas(&self) -> bool { true } diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index d256da28ea..3f5e741dec 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -135,6 +135,16 @@ impl Dialect for ClickHouseDialect { true } + /// See + fn supports_hash_comment(&self) -> bool { + true + } + + /// See + fn requires_hash_comment_space(&self) -> bool { + true + } + /// See fn supports_optimize_table(&self) -> bool { true diff --git a/src/dialect/hive.rs b/src/dialect/hive.rs index 1a26f56ba2..1a7d6de9e3 100644 --- a/src/dialect/hive.rs +++ b/src/dialect/hive.rs @@ -27,6 +27,10 @@ impl Dialect for HiveDialect { (ch == '"') || (ch == '`') } + fn supports_hash_comment(&self) -> bool { + true + } + /// See fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('`') diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index f7c54a9b76..2bf1f6d6ef 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1291,6 +1291,18 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if the dialect supports `#` as the start of a single line comment, + /// e.g. `SELECT 1 # comment` + fn supports_hash_comment(&self) -> bool { + false + } + + /// Returns true if `#` starts a single line comment only when followed by an ASCII + /// space or `!`, so `# comment` and `#! comment` are comments but `#comment` is not. + fn requires_hash_comment_space(&self) -> bool { + false + } + /// Returns true if the dialect supports optimizer hints in multiline comments /// e.g. `/*!50110 KEY_BLOCK_SIZE = 1024*/` fn supports_multiline_comment_hints(&self) -> bool { diff --git a/src/dialect/mysql.rs b/src/dialect/mysql.rs index b85b43f58b..2afee112f1 100644 --- a/src/dialect/mysql.rs +++ b/src/dialect/mysql.rs @@ -164,6 +164,10 @@ impl Dialect for MySqlDialect { true } + fn supports_hash_comment(&self) -> bool { + true + } + fn supports_match_against(&self) -> bool { true } diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index bc954a45b5..ce4cde4b00 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -140,6 +140,10 @@ impl Dialect for SnowflakeDialect { ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_' } + fn supports_hash_comment(&self) -> bool { + true + } + /// See fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('"') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2adf0aa99f..4e76be8417 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -40,16 +40,13 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +use crate::ast::{DollarQuotedString, QuoteDelimitedString}; use crate::dialect::Dialect; use crate::dialect::{ BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect, SnowflakeDialect, }; use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX}; -use crate::{ - ast::{DollarQuotedString, QuoteDelimitedString}, - dialect::HiveDialect, -}; /// SQL Token enumeration #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -1736,9 +1733,15 @@ impl<'a> Tokenizer<'a> { } '{' => self.consume_and_return(chars, Token::LBrace), '}' => self.consume_and_return(chars, Token::RBrace), - '#' if dialect_of!(self is SnowflakeDialect | BigQueryDialect | MySqlDialect | HiveDialect) => + '#' if self.dialect.supports_hash_comment() + && (!self.dialect.requires_hash_comment_space() + || chars + .peekable + .clone() + .nth(1) + .is_some_and(|c| c == ' ' || c == '!')) => { - chars.next(); // consume the '#', starting a snowflake single-line comment + chars.next(); // consume the '#', starting a single-line comment let comment = self.tokenize_single_line_comment(chars); Ok(Some(Token::Whitespace(Whitespace::SingleLineComment { prefix: "#".to_owned(), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 1959bcd345..97a484ef7d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -39,8 +39,7 @@ use sqlparser::dialect::{ }; use sqlparser::keywords::{Keyword, ALL_KEYWORDS}; use sqlparser::parser::{Parser, ParserError, ParserOptions}; -use sqlparser::tokenizer::Tokenizer; -use sqlparser::tokenizer::{Location, Span}; +use sqlparser::tokenizer::{Location, Span, Token, Tokenizer, Whitespace}; use test_utils::{ all_dialects, all_dialects_where, all_dialects_with_options, alter_table_op, assert_eq_vec, call, expr_from_projection, join, number, only, table, table_alias, table_from_name, @@ -13253,22 +13252,71 @@ fn test_release_savepoint() { #[test] fn test_comment_hash_syntax() { - let dialects = TestedDialects::new(vec![ - Box::new(BigQueryDialect {}), - Box::new(SnowflakeDialect {}), - Box::new(MySqlDialect {}), - Box::new(HiveDialect {}), - ]); let sql = r#" # comment SELECT a, b, c # , d, e FROM T ####### comment ################# - WHERE true + WHERE true # it's a comment # comment "#; let canonical = "SELECT a, b, c FROM T WHERE true"; + all_dialects_where(|d| d.supports_hash_comment() && !d.requires_hash_comment_space()) + .verified_only_select_with_canonical(sql, canonical); + + // Elsewhere `#` is an operator, so the apostrophe starts an unterminated string. + assert_eq!( + all_dialects_where(|d| !d.supports_hash_comment()) + .parse_sql_statements("SELECT 1 # it's a comment") + .unwrap_err(), + ParserError::TokenizerError( + "Unterminated string literal at Line: 1, Column: 14".to_string() + ) + ); +} + +#[test] +fn test_comment_hash_requires_space() { + let dialects = all_dialects_where(|d| d.requires_hash_comment_space()); + + let sql = r#" + # comment + #! shebang comment + SELECT a, b, c # , d, e + FROM T + WHERE true #! it's a comment + "#; + let canonical = "SELECT a, b, c FROM T WHERE true"; dialects.verified_only_select_with_canonical(sql, canonical); + + // Without a space or `!` after it, `#` is an operator token, which the parser rejects. + for sql in [ + "SELECT 1 #comment", + "SELECT 1 ####### comment", + "SELECT 1 #", + "SELECT 1 #\tcomment", + ] { + assert_eq!( + dialects.parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError("No infix parser for token Sharp".to_string()), + "{sql}" + ); + } + dialects.tokenizes_to( + "SELECT 1 ### comment", + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::Number("1".to_string(), false), + Token::Whitespace(Whitespace::Space), + Token::Sharp, + Token::Sharp, + Token::Whitespace(Whitespace::SingleLineComment { + prefix: "#".to_string(), + comment: " comment".to_string(), + }), + ], + ); } #[test]