replaces the hard-coded dialect list in the tokenizer with a supports_hash_comment dialect method

fuzz/fuzz_seeds/clickhouse_hash_comment.sql

@@ -0,0 +1,48 @@
+#!/usr/bin/clickhouse-local --queries-file
+# leading line comment
+#! shebang style comment
+SELECT 1 # trailing comment
+;
+SELECT a, b # , c
+FROM t # table
+WHERE x = 1 # it's a comment with an apostrophe
+;
+SELECT # comment right after SELECT
+    a,
+    #! comment between select items
+    b
+FROM t;
+SELECT 1 #!no space after bang, the semicolon is part of the comment;
+;
+SELECT 1 #   several spaces then comment
+;
+SELECT '# not a comment', "# not a comment", `# not a comment` FROM t # real comment
+;
+SELECT a FROM t WHERE s = 'a # b' # comment containing 'quotes' and "double quotes"
+;
+SELECT a FROM t -- dash comment
+# hash comment
+/* block comment */
+#! shebang comment
+WHERE a = 1;
+SELECT /* # not a comment */ 1 # comment with /* block */ inside
+;
+SELECT 1 # comment with -- inside
+;
+SELECT 1 # comment with # another hash # and more
+;
+SELECT * FROM t PREWHERE x GLOBAL IN (1, 2) # after prewhere
+SETTINGS max_threads = 1 # after settings
+;
+INSERT INTO t2 SELECT * FROM t # comment in insert
+;
+CREATE TABLE t (a Int32, # comment after column
+    b String # comment after last column
+) ENGINE = MergeTree ORDER BY a # comment after order by
+;
+WITH c AS (SELECT 1 AS v) # comment after cte
+SELECT * FROM c;
+SELECT arrayMap(x -> x + 1, # comment inside call
+    [1, 2]) # comment after call
+;
+SELECT 1 # comment at end of file without newline

src/dialect/bigquery.rs

@@ -72,6 +72,10 @@ impl Dialect for BigQueryDialect {
         Some('`')
     }
 
+    fn supports_hash_comment(&self) -> bool {
+        true
+    }
+
     fn supports_projection_trailing_commas(&self) -> bool {
         true
     }

src/dialect/clickhouse.rs

@@ -135,6 +135,16 @@ impl Dialect for ClickHouseDialect {
         true
     }
 
+    /// See <https://clickhouse.com/docs/sql-reference/syntax#comments>
+    fn supports_hash_comment(&self) -> bool {
+        true
+    }
+
+    /// See <https://github.com/ClickHouse/ClickHouse/blob/master/src/Parsers/Lexer.cpp>
+    fn requires_hash_comment_space(&self) -> bool {
+        true
+    }
+
     /// See <https://clickhouse.com/docs/en/sql-reference/statements/optimize>
     fn supports_optimize_table(&self) -> bool {
         true

src/dialect/hive.rs

@@ -27,6 +27,10 @@ impl Dialect for HiveDialect {
         (ch == '"') || (ch == '`')
     }
 
+    fn supports_hash_comment(&self) -> bool {
+        true
+    }
+
     /// See <https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-RulesforColumnNames>
     fn identifier_quote_style(&self, _identifier: &str) -> Option<char> {
         Some('`')

src/dialect/mod.rs

@@ -1291,6 +1291,18 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if the dialect supports `#` as the start of a single line comment,
+    /// e.g. `SELECT 1 # comment`
+    fn supports_hash_comment(&self) -> bool {
+        false
+    }
+
+    /// Returns true if `#` starts a single line comment only when followed by an ASCII
+    /// space or `!`, so `# comment` and `#! comment` are comments but `#comment` is not.
+    fn requires_hash_comment_space(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialect supports optimizer hints in multiline comments
     /// e.g. `/*!50110 KEY_BLOCK_SIZE = 1024*/`
     fn supports_multiline_comment_hints(&self) -> bool {

src/dialect/mysql.rs

@@ -164,6 +164,10 @@ impl Dialect for MySqlDialect {
         true
     }
 
+    fn supports_hash_comment(&self) -> bool {
+        true
+    }
+
     fn supports_match_against(&self) -> bool {
         true
     }

src/dialect/snowflake.rs

@@ -140,6 +140,10 @@ impl Dialect for SnowflakeDialect {
         ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_'
     }
 
+    fn supports_hash_comment(&self) -> bool {
+        true
+    }
+
     /// See <https://docs.snowflake.com/en/sql-reference/identifiers-syntax>
     fn identifier_quote_style(&self, _identifier: &str) -> Option<char> {
         Some('"')

src/tokenizer.rs

@@ -40,16 +40,13 @@ use serde::{Deserialize, Serialize};
 #[cfg(feature = "visitor")]
 use sqlparser_derive::{Visit, VisitMut};
 
+use crate::ast::{DollarQuotedString, QuoteDelimitedString};
 use crate::dialect::Dialect;
 use crate::dialect::{
     BigQueryDialect, DuckDbDialect, GenericDialect, MySqlDialect, PostgreSqlDialect,
     SnowflakeDialect,
 };
 use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
-use crate::{
-    ast::{DollarQuotedString, QuoteDelimitedString},
-    dialect::HiveDialect,
-};
 
 /// SQL Token enumeration
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
@@ -1736,9 +1733,15 @@ impl<'a> Tokenizer<'a> {
                 }
                 '{' => self.consume_and_return(chars, Token::LBrace),
                 '}' => self.consume_and_return(chars, Token::RBrace),
-                '#' if dialect_of!(self is SnowflakeDialect | BigQueryDialect | MySqlDialect | HiveDialect) =>
+                '#' if self.dialect.supports_hash_comment()
+                    && (!self.dialect.requires_hash_comment_space()
+                        || chars
+                            .peekable
+                            .clone()
+                            .nth(1)
+                            .is_some_and(|c| c == ' ' || c == '!')) =>
                 {
-                    chars.next(); // consume the '#', starting a snowflake single-line comment
+                    chars.next(); // consume the '#', starting a single-line comment
                     let comment = self.tokenize_single_line_comment(chars);
                     Ok(Some(Token::Whitespace(Whitespace::SingleLineComment {
                         prefix: "#".to_owned(),

tests/sqlparser_common.rs

@@ -39,8 +39,7 @@ use sqlparser::dialect::{
 };
 use sqlparser::keywords::{Keyword, ALL_KEYWORDS};
 use sqlparser::parser::{Parser, ParserError, ParserOptions};
-use sqlparser::tokenizer::Tokenizer;
-use sqlparser::tokenizer::{Location, Span};
+use sqlparser::tokenizer::{Location, Span, Token, Tokenizer, Whitespace};
 use test_utils::{
     all_dialects, all_dialects_where, all_dialects_with_options, alter_table_op, assert_eq_vec,
     call, expr_from_projection, join, number, only, table, table_alias, table_from_name,
@@ -13253,22 +13252,71 @@ fn test_release_savepoint() {
 
 #[test]
 fn test_comment_hash_syntax() {
-    let dialects = TestedDialects::new(vec![
-        Box::new(BigQueryDialect {}),
-        Box::new(SnowflakeDialect {}),
-        Box::new(MySqlDialect {}),
-        Box::new(HiveDialect {}),
-    ]);
     let sql = r#"
     # comment
     SELECT a, b, c # , d, e
     FROM T
     ####### comment #################
-    WHERE true
+    WHERE true # it's a comment
     # comment
     "#;
     let canonical = "SELECT a, b, c FROM T WHERE true";
+    all_dialects_where(|d| d.supports_hash_comment() && !d.requires_hash_comment_space())
+        .verified_only_select_with_canonical(sql, canonical);
+
+    // Elsewhere `#` is an operator, so the apostrophe starts an unterminated string.
+    assert_eq!(
+        all_dialects_where(|d| !d.supports_hash_comment())
+            .parse_sql_statements("SELECT 1 # it's a comment")
+            .unwrap_err(),
+        ParserError::TokenizerError(
+            "Unterminated string literal at Line: 1, Column: 14".to_string()
+        )
+    );
+}
+
+#[test]
+fn test_comment_hash_requires_space() {
+    let dialects = all_dialects_where(|d| d.requires_hash_comment_space());
+
+    let sql = r#"
+    # comment
+    #! shebang comment
+    SELECT a, b, c # , d, e
+    FROM T
+    WHERE true #! it's a comment
+    "#;
+    let canonical = "SELECT a, b, c FROM T WHERE true";
     dialects.verified_only_select_with_canonical(sql, canonical);
+
+    // Without a space or `!` after it, `#` is an operator token, which the parser rejects.
+    for sql in [
+        "SELECT 1 #comment",
+        "SELECT 1 ####### comment",
+        "SELECT 1 #",
+        "SELECT 1 #\tcomment",
+    ] {
+        assert_eq!(
+            dialects.parse_sql_statements(sql).unwrap_err(),
+            ParserError::ParserError("No infix parser for token Sharp".to_string()),
+            "{sql}"
+        );
+    }
+    dialects.tokenizes_to(
+        "SELECT 1 ### comment",
+        vec![
+            Token::make_keyword("SELECT"),
+            Token::Whitespace(Whitespace::Space),
+            Token::Number("1".to_string(), false),
+            Token::Whitespace(Whitespace::Space),
+            Token::Sharp,
+            Token::Sharp,
+            Token::Whitespace(Whitespace::SingleLineComment {
+                prefix: "#".to_string(),
+                comment: " comment".to_string(),
+            }),
+        ],
+    );
 }
 
 #[test]