Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,9 @@ public class BigQueryConnection extends BigQueryNoOpsConnection {
BigQueryJdbcUrlUtility.OAUTH_SA_IMPERSONATION_TOKEN_LIFETIME_PROPERTY_NAME,
BigQueryJdbcUrlUtility.OAUTH_SA_EMAIL_PROPERTY_NAME,
BigQueryJdbcUrlUtility.OAUTH_PVT_KEY_PATH_PROPERTY_NAME,
BigQueryJdbcUrlUtility.OAUTH_CLIENT_ID_PROPERTY_NAME)
BigQueryJdbcUrlUtility.OAUTH_CLIENT_ID_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_DIAGNOSTIC_TELEMETRY_PROPERTY_NAME,
BigQueryJdbcUrlUtility.ENABLE_GRAPH_METADATA_PROPERTY_NAME)
.build();
private HeaderProvider headerProvider;
BigQueryReadClient bigQueryReadClient = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
Expand Down Expand Up @@ -103,6 +104,12 @@ class BigQueryDatabaseMetaData implements DatabaseMetaData {
private static final int DEFAULT_QUEUE_CAPACITY = 5000;
private static final String GET_EXPORTED_KEYS_SQL = "DatabaseMetaData_GetExportedKeys.sql";
private static String exportedKeysSqlContent;
private static final String GET_GRAPH_TABLES_REGION_SQL =
"DatabaseMetaData_GetGraphTables_Region.sql";
private static String graphTablesRegionSqlContent;
private static final String GET_GRAPH_TABLES_DATASET_SQL =
"DatabaseMetaData_GetGraphTables_Dataset.sql";
private static String graphTablesDatasetSqlContent;
// Declared package-private for testing.
static final String GOOGLE_SQL_QUOTED_IDENTIFIER = "`";
// Does not include SQL:2003 Keywords as per JDBC spec.
Expand Down Expand Up @@ -138,6 +145,15 @@ class BigQueryDatabaseMetaData implements DatabaseMetaData {
static final String GOOGLE_SQL_CATALOG_SEPARATOR = ".";
static final int GOOGLE_SQL_MAX_COL_NAME_LEN = 300;
static final int GOOGLE_SQL_MAX_COLS_PER_TABLE = 10000;
private static final String[] DEFAULT_TABLE_TYPES = {
"EXTERNAL", "MATERIALIZED VIEW", "SNAPSHOT", "TABLE", "VIEW"
};
private static final String[] TABLE_TYPES_WITH_GRAPH = {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we update it to be DEFAULT_TABLE_TYPES + {"GRAPH TABLE"} in some way?

"EXTERNAL", "GRAPH TABLE", "MATERIALIZED VIEW", "SNAPSHOT", "TABLE", "VIEW"
};
static final String GRAPH_TABLE_TYPE = "GRAPH TABLE";
private static final String DEFAULT_GRAPH_REGION = "us";
private static final Pattern VALID_IDENTIFIER_PATTERN = Pattern.compile("^[A-Za-z0-9_\\-:.]+$");

String URL;
BigQueryConnection connection;
Expand Down Expand Up @@ -1308,19 +1324,37 @@ private ResultSet getTablesImpl(
() -> {
final List<FieldValueList> collectedResults =
Collections.synchronizedList(new ArrayList<>());
processTargetTablesConcurrently(
targetDatasets,
boolean onlyGraphTablesRequested =
requestedTypes != null
&& requestedTypes.size() == 1
&& requestedTypes.contains(GRAPH_TABLE_TYPE);
boolean isExactGraphTableName =
tableNamePattern != null
&& !tableNamePattern.contains("%")
&& tableNamePattern.contains(".");

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add a comment explaining this assumption that if table name contains ., we'll expect it to be graph

if (!isExactGraphTableName && !onlyGraphTablesRequested) {
processTargetTablesConcurrently(
targetDatasets,
tableNamePattern,
true,
false,
false,
collectedResults,
resultSchemaFields,
(bqTable, results, fields) -> {
processTableInfo(bqTable, requestedTypes, results, fields);
},
TableOption.fields(
TableField.TABLE_REFERENCE, TableField.TYPE, TableField.DESCRIPTION));
}

appendGraphTableRows(
effectiveCatalog,
effectiveSchemaPattern,
tableNamePattern,
true,
false,
false,
requestedTypes,
collectedResults,
resultSchemaFields,
(bqTable, results, fields) -> {
processTableInfo(bqTable, requestedTypes, results, fields);
},
TableOption.fields(
TableField.TABLE_REFERENCE, TableField.TYPE, TableField.DESCRIPTION));
resultSchemaFields);

Comparator<FieldValueList> comparator = defineGetTablesComparator(resultSchemaFields);
sortResults(collectedResults, comparator, "getTables", LOG);
Expand Down Expand Up @@ -1441,6 +1475,120 @@ Comparator<FieldValueList> defineGetTablesComparator(FieldList resultSchemaField
Comparator.nullsFirst(String::compareTo));
}

static String quoteIdentifier(String identifier) throws SQLException {
if (identifier == null || !VALID_IDENTIFIER_PATTERN.matcher(identifier).matches()) {
throw new BigQueryJdbcException(
String.format("Invalid BigQuery identifier: '%s'", identifier));
}
return GOOGLE_SQL_QUOTED_IDENTIFIER + identifier + GOOGLE_SQL_QUOTED_IDENTIFIER;
}

boolean shouldSkipGraphTables(String tableNamePattern, Set<String> requestedTypes) {
if (this.connection == null || !this.connection.isEnableGraphMetadata()) {
return true;
}
if (requestedTypes != null && !requestedTypes.contains(GRAPH_TABLE_TYPE)) {
return true;
}
if (tableNamePattern != null
&& !tableNamePattern.contains("%")
&& !tableNamePattern.contains(".")) {
Comment thread
Neenu1995 marked this conversation as resolved.
Resolved
return true;
}
return false;
}

String resolveGraphRegion() throws SQLException {
String location = this.connection != null ? this.connection.getLocation() : null;
if (location == null || location.trim().isEmpty()) {
location = DEFAULT_GRAPH_REGION;
Comment thread
Neenu1995 marked this conversation as resolved.
Resolved
LOG.info(String.format("Deafulting to Graph Location %s", DEFAULT_GRAPH_REGION));
}
String normalized = location.trim().toLowerCase(Locale.US);
return normalized.startsWith("region-") ? normalized : "region-" + normalized;
}

void appendGraphTableRows(
String effectiveCatalog,
String effectiveSchemaPattern,
String tableNamePattern,
Set<String> requestedTypes,
List<FieldValueList> collectedResults,
FieldList resultSchemaResults)
throws SQLException {
if (shouldSkipGraphTables(tableNamePattern, requestedTypes)) {
return;
}

LOG.info("Appending Graph table rows");

List<String> targetCatalogs =
effectiveCatalog != null
? Collections.singletonList(effectiveCatalog)
: getAccessibleCatalogNames();
boolean isExactDataset =
effectiveSchemaPattern != null && !needsListing(effectiveSchemaPattern);
String effectiveTablePattern = (tableNamePattern == null) ? "%" : tableNamePattern;

for (String project : targetCatalogs) {
String formattedSql;
if (isExactDataset) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I'd rather do it this way, easier to see what's changing

formattedSql =
            replaceSqlParameters(
                getGraphTablesRegionSqlContent(),
                quoteIdentifier(project),
                isExactDataset ? quoteIdentifier(effectiveSchemaPattern) : quoteIdentifier(resolveGraphRegion()));

formattedSql =
replaceSqlParameters(
getGraphTablesDatasetSqlContent(),
quoteIdentifier(project),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per comment in SQL, rather than escaping identifiers here, better to do it at SQL file level

quoteIdentifier(effectiveSchemaPattern));
} else {
formattedSql =
replaceSqlParameters(
getGraphTablesRegionSqlContent(),
quoteIdentifier(project),
quoteIdentifier(resolveGraphRegion()));
}

try (PreparedStatement statement = this.connection.prepareStatement(formattedSql)) {
if (isExactDataset) {
statement.setString(1, effectiveTablePattern);
} else {
statement.setString(1, effectiveSchemaPattern == null ? "%" : effectiveSchemaPattern);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can have single SQL query, having an extra condition schema LIKE "%" is unlikely to have any perf implication (other than that they seem identical);

statement.setString(2, effectiveTablePattern);
}

try (ResultSet rs = statement.executeQuery()) {
while (rs.next()) {
String tableCat = rs.getString("TABLE_CAT");
String tableSchem = rs.getString("TABLE_SCHEM");
String tableName = rs.getString("TABLE_NAME");
String sourceTable = rs.getString("SOURCE_TABLE");
String remarks = sourceTable != null ? "Source: " + sourceTable : null;

List<FieldValue> values = new ArrayList<>(resultSchemaResults.size());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add inline comments what maps to what?

values.add(createStringFieldValue(tableCat));
values.add(createStringFieldValue(tableSchem));
values.add(createStringFieldValue(tableName));
values.add(createStringFieldValue(GRAPH_TABLE_TYPE));
values.add(createStringFieldValue(remarks));
values.add(createNullFieldValue());
values.add(createNullFieldValue());
values.add(createNullFieldValue());
values.add(createNullFieldValue());
values.add(createNullFieldValue());
collectedResults.add(FieldValueList.of(values, resultSchemaResults));
Comment thread
Neenu1995 marked this conversation as resolved.
Resolved
}
}
} catch (SQLException e) {
if (e.getMessage() != null && e.getMessage().contains("Not found")) {
LOG.info(
"Dataset or project not found while querying PROPERTY_GRAPHS in project '%s'."
+ " Skipping.",
project);
} else {
throw e;
}
}
Comment thread
Neenu1995 marked this conversation as resolved.
Resolved
}
}

@Override
public ResultSet getSchemas() throws SQLException {
LOG.info("getSchemas() called");
Expand Down Expand Up @@ -1488,7 +1636,8 @@ public ResultSet getTableTypes() throws SQLException {
LOG.info("getTableTypes() called");

final Schema tableTypesSchema = defineGetTableTypesSchema();
final List<FieldValueList> tableTypeRows = prepareGetTableTypesRows(tableTypesSchema);
final List<FieldValueList> tableTypeRows =
prepareGetTableTypesRows(tableTypesSchema, this.connection.isEnableGraphMetadata());

BlockingQueue<BigQueryFieldValueListWrapper> queue =
fetchAndPopulateQueue(() -> tableTypeRows, tableTypesSchema.getFields());
Expand All @@ -1504,9 +1653,13 @@ static Schema defineGetTableTypesSchema() {
}

static List<FieldValueList> prepareGetTableTypesRows(Schema schema) {
final String[] tableTypes = {"EXTERNAL", "MATERIALIZED VIEW", "SNAPSHOT", "TABLE", "VIEW"};
List<FieldValueList> rows = new ArrayList<>(tableTypes.length);
return prepareGetTableTypesRows(schema, false);
}

static List<FieldValueList> prepareGetTableTypesRows(Schema schema, boolean includeGraphTables) {
final String[] tableTypes = includeGraphTables ? TABLE_TYPES_WITH_GRAPH : DEFAULT_TABLE_TYPES;
FieldList schemaFields = schema.getFields();
List<FieldValueList> rows = new ArrayList<>(tableTypes.length);

for (String typeName : tableTypes) {
FieldValue fieldValue = FieldValue.of(FieldValue.Attribute.PRIMITIVE, typeName);
Expand Down Expand Up @@ -4853,6 +5006,20 @@ private static synchronized String getExportedKeysSqlContent() {
return exportedKeysSqlContent;
}

static synchronized String getGraphTablesDatasetSqlContent() {
if (graphTablesDatasetSqlContent == null) {
graphTablesDatasetSqlContent = readSqlFromFile(GET_GRAPH_TABLES_DATASET_SQL);
}
return graphTablesDatasetSqlContent;
}

static synchronized String getGraphTablesRegionSqlContent() {
if (graphTablesRegionSqlContent == null) {
graphTablesRegionSqlContent = readSqlFromFile(GET_GRAPH_TABLES_REGION_SQL);
}
return graphTablesRegionSqlContent;
}

static String readSqlFromFile(String filename) {
try (InputStream in = BigQueryDatabaseMetaData.class.getResourceAsStream(filename)) {
if (in == null) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

SELECT
property_graph_catalog AS TABLE_CAT,
property_graph_schema AS TABLE_SCHEM,
CONCAT(property_graph_name, '.', JSON_VALUE(t, '$.name')) AS TABLE_NAME,
CONCAT(
JSON_VALUE(t, '$.dataSourceTable.projectId'), '.',
JSON_VALUE(t, '$.dataSourceTable.datasetId'), '.',
JSON_VALUE(t, '$.dataSourceTable.tableId')) AS SOURCE_TABLE
FROM %1$s.%2$s.INFORMATION_SCHEMA.PROPERTY_GRAPHS,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add escape ``` characters for the table name

UNNEST(ARRAY_CONCAT(
IFNULL(JSON_QUERY_ARRAY(property_graph_metadata_json, '$.nodeTables'), []),
IFNULL(JSON_QUERY_ARRAY(property_graph_metadata_json, '$.edgeTables'), []))) AS t
WHERE LOWER(CONCAT(property_graph_name, '.', JSON_VALUE(t, '$.name'))) LIKE LOWER(?)
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

SELECT
property_graph_catalog AS TABLE_CAT,
property_graph_schema AS TABLE_SCHEM,
CONCAT(property_graph_name, '.', JSON_VALUE(t, '$.name')) AS TABLE_NAME,
CONCAT(
JSON_VALUE(t, '$.dataSourceTable.projectId'), '.',
JSON_VALUE(t, '$.dataSourceTable.datasetId'), '.',
JSON_VALUE(t, '$.dataSourceTable.tableId')) AS SOURCE_TABLE
FROM %1$s.%2$s.INFORMATION_SCHEMA.PROPERTY_GRAPHS,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Escape table name

UNNEST(ARRAY_CONCAT(
IFNULL(JSON_QUERY_ARRAY(property_graph_metadata_json, '$.nodeTables'), []),
IFNULL(JSON_QUERY_ARRAY(property_graph_metadata_json, '$.edgeTables'), []))) AS t
WHERE LOWER(property_graph_schema) LIKE LOWER(?)
AND LOWER(CONCAT(property_graph_name, '.', JSON_VALUE(t, '$.name'))) LIKE LOWER(?)
Loading
Loading