diff --git a/services/libs/tinybird/datasources/insights_projects_populated_ds.datasource b/services/libs/tinybird/datasources/insights_projects_populated_ds.datasource index 7b8c02152c..4003f377b2 100644 --- a/services/libs/tinybird/datasources/insights_projects_populated_ds.datasource +++ b/services/libs/tinybird/datasources/insights_projects_populated_ds.datasource @@ -25,6 +25,10 @@ DESCRIPTION > - `status` is the status of the project ex: active or archived - `maturity` is the project maturity stage (e.g. Sandbox, Incubating, Graduated). Empty string when not set. - `lastVulnerabilityScanStatus` is the aggregated vulnerability scan status across all project repositories (`LowCardinality(Nullable(String))`). Possible values: `'success'` (all scans passed), `'no_packages_found'` (no dependency packages detected), `'failure'` (at least one scan failed), `'running'` (scan in progress). NULL when no scan records exist for the project's repositories. + - `healthScoreV2` is the Health Score v2 composite (0-100), copied from `project_insights_copy_ds`. NULL when fewer than 2 categories have data, and for archived projects and `korg`. + - `healthLabel` is the Health Score v2 band ('excellent' to 'critical'), computed on the normalized score. NULL when `healthScoreV2` is NULL. + - `healthScoreV2Normalized` is `healthScoreV2 * 100 / healthMaxScore` (0-100), so partial scores sort on one scale. NULL when `healthScoreV2` or `healthMaxScore` is NULL. + - `healthMaxScore` is the denominator of a partial `healthScoreV2` (60, 65, 75 or 100), copied from `project_insights_copy_ds`. NULL when `healthScoreV2` is NULL. TAGS "Project metadata", "Analytics enrichment" @@ -66,7 +70,11 @@ SCHEMA > `status` String, `maturity` LowCardinality(String), `lastVulnerabilityScanStatus` Nullable(String), - `repoLicenses` Array(Tuple(String, String)) + `repoLicenses` Array(Tuple(String, String)), + `healthScoreV2` Nullable(UInt8), + `healthLabel` Nullable(String), + `healthScoreV2Normalized` Nullable(Float64), + `healthMaxScore` Nullable(UInt8) ENGINE MergeTree ENGINE_PARTITION_KEY toYear(createdAt) diff --git a/services/libs/tinybird/pipes/categories_oss_index.pipe b/services/libs/tinybird/pipes/categories_oss_index.pipe index 7f17a42f32..7ed89df152 100644 --- a/services/libs/tinybird/pipes/categories_oss_index.pipe +++ b/services/libs/tinybird/pipes/categories_oss_index.pipe @@ -34,9 +34,27 @@ DESCRIPTION > Returns only needed columns from insights projects SQL > - SELECT id, name, logoUrl, description, healthScore + SELECT + id, + name, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore FROM insights_projects_populated_ds - GROUP BY id, name, logoUrl, description, healthScore + GROUP BY + id, + name, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore NODE categories_oss_index_collections_deduplicated DESCRIPTION > @@ -80,7 +98,11 @@ SQL > pd.name AS projectName, pd.logoUrl AS projectLogo, pd.description AS description, - pd.healthScore AS healthScore + pd.healthScore AS healthScore, + pd.healthScoreV2 AS healthScoreV2, + pd.healthLabel AS healthLabel, + pd.healthScoreV2Normalized AS healthScoreV2Normalized, + pd.healthMaxScore AS healthMaxScore FROM categories_oss_index_projects_ranked_by_category topProjects JOIN categories_oss_index_projects_deduplicated pd ON pd.id = topProjects.projectId WHERE topProjects.rn <= 5 @@ -160,7 +182,11 @@ SQL > softwareValue, avgScore, healthScore, - description + description, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore ) ) AS topProjects FROM categories_oss_index_projects diff --git a/services/libs/tinybird/pipes/category_groups_oss_index.pipe b/services/libs/tinybird/pipes/category_groups_oss_index.pipe index 136fcb00f5..2418c5ec38 100644 --- a/services/libs/tinybird/pipes/category_groups_oss_index.pipe +++ b/services/libs/tinybird/pipes/category_groups_oss_index.pipe @@ -31,9 +31,29 @@ DESCRIPTION > Returns only columns which we need from insights projects table SQL > - SELECT id, name, logoUrl, description, healthScore, status + SELECT + id, + name, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore, + status FROM insights_projects_populated_ds - GROUP BY id, name, logoUrl, description, healthScore, status + GROUP BY + id, + name, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore, + status NODE category_groups_oss_index_collections_deduplicated DESCRIPTION > @@ -78,6 +98,10 @@ SQL > pd.logoUrl AS projectLogo, pd.description AS description, pd.healthScore AS healthScore, + pd.healthScoreV2 AS healthScoreV2, + pd.healthLabel AS healthLabel, + pd.healthScoreV2Normalized AS healthScoreV2Normalized, + pd.healthMaxScore AS healthMaxScore, pd.status as status FROM category_groups_oss_index_projects_ranked_by_category_group topProjects JOIN category_groups_oss_index_projects_deduplicated pd ON pd.id = topProjects.projectId @@ -163,7 +187,11 @@ SQL > avgScore, healthScore, description, - status + status, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore ) ) AS topProjects FROM category_groups_oss_index_projects diff --git a/services/libs/tinybird/pipes/collections_oss_index.pipe b/services/libs/tinybird/pipes/collections_oss_index.pipe index dfa5daf052..d5396e3b01 100644 --- a/services/libs/tinybird/pipes/collections_oss_index.pipe +++ b/services/libs/tinybird/pipes/collections_oss_index.pipe @@ -38,9 +38,31 @@ DESCRIPTION > Returns only needed columns from insights projects SQL > - SELECT id, name, slug, logoUrl, description, healthScore, status + SELECT + id, + name, + slug, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore, + status FROM insights_projects_populated_ds - GROUP BY id, name, slug, logoUrl, description, healthScore, status + GROUP BY + id, + name, + slug, + logoUrl, + description, + healthScore, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore, + status NODE collections_oss_index_projects_ranked_by_collection DESCRIPTION > @@ -77,6 +99,10 @@ SQL > pd.logoUrl AS projectLogo, pd.description, pd.healthScore, + pd.healthScoreV2, + pd.healthLabel, + pd.healthScoreV2Normalized, + pd.healthMaxScore, pd.status FROM collections_oss_index_projects_ranked_by_collection topProjects JOIN collections_oss_index_projects_deduplicated pd ON pd.id = topProjects.projectId @@ -112,7 +138,11 @@ SQL > healthScore, description, projectSlug, - status + status, + healthScoreV2, + healthLabel, + healthScoreV2Normalized, + healthMaxScore ) ) AS topProjects FROM collections_oss_index_projects diff --git a/services/libs/tinybird/pipes/insightsProjects_filtered.pipe b/services/libs/tinybird/pipes/insightsProjects_filtered.pipe index e8395c8873..03ac77498f 100644 --- a/services/libs/tinybird/pipes/insightsProjects_filtered.pipe +++ b/services/libs/tinybird/pipes/insightsProjects_filtered.pipe @@ -30,6 +30,10 @@ SQL > insights_projects_populated_ds.firstCommitUrl, insights_projects_populated_ds.repoData, insights_projects_populated_ds.healthScore, + insights_projects_populated_ds.healthScoreV2, + insights_projects_populated_ds.healthLabel, + insights_projects_populated_ds.healthScoreV2Normalized, + insights_projects_populated_ds.healthMaxScore, insights_projects_populated_ds.communityPlatforms, insights_projects_populated_ds.communityKeywords, insights_projects_populated_ds.communityLanguages, diff --git a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe index d874ab6efe..abaa0c9b3d 100644 --- a/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe +++ b/services/libs/tinybird/pipes/insights_projects_populated_copy.pipe @@ -120,6 +120,20 @@ NODE insights_projects_populated_copy_health_score_deduplicated SQL > SELECT id, overallScore as healthScore FROM health_score_copy_ds +NODE insights_projects_populated_copy_health_v2 +DESCRIPTION > + Health Score v2 per project from project_insights_copy_ds, plus the score normalized to a 0-100 scale + +SQL > + SELECT + id, + healthScoreV2, + healthLabel, + if(healthMaxScore > 0, healthScoreV2 * 100.0 / healthMaxScore, NULL) AS healthScoreV2Normalized, + healthMaxScore + FROM project_insights_copy_ds + WHERE type = 'project' + NODE archived_excluded_repositories DESCRIPTION > Get the archived and excluded repositories for each segment and insightsProject @@ -244,7 +258,13 @@ SQL > any (insights_projects_populated_copy_mentions.communityLanguages) as communityLanguages, any (segments.status) as status, any (segments.maturity) as maturity, - any (last_vulnerability_scan_status.lastVulnerabilityScanStatus) as lastVulnerabilityScanStatus + any (last_vulnerability_scan_status.lastVulnerabilityScanStatus) as lastVulnerabilityScanStatus, + any (insights_projects_populated_copy_health_v2.healthScoreV2) as healthScoreV2, + any (insights_projects_populated_copy_health_v2.healthLabel) as healthLabel, + any ( + insights_projects_populated_copy_health_v2.healthScoreV2Normalized + ) as healthScoreV2Normalized, + any (insights_projects_populated_copy_health_v2.healthMaxScore) as healthMaxScore FROM insightsProjects FINAL LEFT JOIN insights_projects_populated_copy_collections_slugs @@ -274,6 +294,9 @@ SQL > last_vulnerability_scan_status ON last_vulnerability_scan_status.insightsProjectId = insightsProjects.id LEFT JOIN segments ON segments.id = insightsProjects.segmentId + LEFT JOIN + insights_projects_populated_copy_health_v2 + ON insights_projects_populated_copy_health_v2.id = insightsProjects.id WHERE isNull (insightsProjects.deletedAt) AND insightsProjects.enabled = 1 group by 1 diff --git a/services/libs/tinybird/pipes/project_insights_copy.pipe b/services/libs/tinybird/pipes/project_insights_copy.pipe index 02c2cec57c..ae526a62e7 100644 --- a/services/libs/tinybird/pipes/project_insights_copy.pipe +++ b/services/libs/tinybird/pipes/project_insights_copy.pipe @@ -38,21 +38,29 @@ SQL > NODE project_insights_copy_dependency_metrics DESCRIPTION > - Get contributor and organization dependency metrics from health_score_copy_ds + Get the v1 category percentages from health_score_copy_ds SQL > - SELECT - slug, - contributorDependencyCount, - contributorDependencyPercentage, - organizationDependencyCount, - organizationDependencyPercentage, - contributorPercentage, - popularityPercentage, - developmentPercentage, - securityPercentage + SELECT slug, contributorPercentage, popularityPercentage, developmentPercentage, securityPercentage FROM health_score_copy_ds +NODE project_insights_copy_contributor_dependency +DESCRIPTION > + Contributor dependency, last 365 days; segmentId aliased to avoid the USING (segmentId) clash + +SQL > + SELECT segmentId AS dependencySegmentId, contributorDependencyCount, contributorDependencyPercentage + FROM health_score_contributor_dependency + +NODE project_insights_copy_organization_dependency +DESCRIPTION > + Organization dependency, last 365 days; segmentId aliased to avoid the USING (segmentId) clash + +SQL > + SELECT + segmentId AS dependencySegmentId, organizationDependencyCount, organizationDependencyPercentage + FROM health_score_organization_dependency + NODE project_insights_copy_achievements DESCRIPTION > Aggregate leaderboard achievements per project from leaderboards_copy_ds (latest snapshot) @@ -281,10 +289,10 @@ SQL > base.contributorCount AS contributorCount, base.organizationCount AS organizationCount, base.softwareValue AS softwareValue, - COALESCE(dep.contributorDependencyCount, 0) AS contributorDependencyCount, - COALESCE(dep.contributorDependencyPercentage, 0) AS contributorDependencyPercentage, - COALESCE(dep.organizationDependencyCount, 0) AS organizationDependencyCount, - COALESCE(dep.organizationDependencyPercentage, 0) AS organizationDependencyPercentage, + COALESCE(cdep.contributorDependencyCount, 0) AS contributorDependencyCount, + COALESCE(cdep.contributorDependencyPercentage, 0) AS contributorDependencyPercentage, + COALESCE(odep.organizationDependencyCount, 0) AS organizationDependencyCount, + COALESCE(odep.organizationDependencyPercentage, 0) AS organizationDependencyPercentage, COALESCE(ach.achievements, []) AS achievements, base.healthScore AS healthScore, dep.contributorPercentage AS contributorHealthScore, @@ -349,6 +357,12 @@ SQL > if(base.slug = 'korg' OR isArchived, NULL, hv2.healthMaxScore) AS healthMaxScore FROM project_insights_copy_base_projects AS base LEFT JOIN project_insights_copy_dependency_metrics AS dep ON base.slug = dep.slug + LEFT JOIN + project_insights_copy_contributor_dependency AS cdep + ON base.segmentId = cdep.dependencySegmentId + LEFT JOIN + project_insights_copy_organization_dependency AS odep + ON base.segmentId = odep.dependencySegmentId LEFT JOIN project_insights_copy_achievements AS ach ON base.slug = ach.slug LEFT JOIN project_insights_copy_period_metrics AS pm USING (segmentId) LEFT JOIN project_insights_copy_star_deltas AS sd ON base.segmentId = sd.segmentId @@ -409,6 +423,22 @@ SQL > WHERE timestamp >= now() - INTERVAL 730 DAY AND timestamp <= now() GROUP BY channel +NODE project_insights_copy_repo_contributor_dependency +DESCRIPTION > + Repository contributor dependency, last 365 days + +SQL > + SELECT channel, contributorDependencyCount, contributorDependencyPercentage + FROM repo_health_score_contributor_dependency + +NODE project_insights_copy_repo_organization_dependency +DESCRIPTION > + Repository organization dependency, last 365 days + +SQL > + SELECT channel, organizationDependencyCount, organizationDependencyPercentage + FROM repo_health_score_organization_dependency + NODE project_insights_copy_repo_results DESCRIPTION > Join all repository insights together @@ -426,10 +456,10 @@ SQL > base.contributorCount AS contributorCount, base.organizationCount AS organizationCount, base.softwareValue AS softwareValue, - COALESCE(hs.contributorDependencyCount, 0) AS contributorDependencyCount, - COALESCE(hs.contributorDependencyPercentage, 0) AS contributorDependencyPercentage, - COALESCE(hs.organizationDependencyCount, 0) AS organizationDependencyCount, - COALESCE(hs.organizationDependencyPercentage, 0) AS organizationDependencyPercentage, + COALESCE(cdep.contributorDependencyCount, 0) AS contributorDependencyCount, + COALESCE(cdep.contributorDependencyPercentage, 0) AS contributorDependencyPercentage, + COALESCE(odep.organizationDependencyCount, 0) AS organizationDependencyCount, + COALESCE(odep.organizationDependencyPercentage, 0) AS organizationDependencyPercentage, CAST([] AS Array(Tuple(String, UInt64, UInt64))) AS achievements, hs.overallScore AS healthScore, hs.contributorPercentage AS contributorHealthScore, @@ -481,20 +511,37 @@ SQL > CAST(NULL AS Nullable(UInt8)) AS healthMaxScore FROM project_insights_copy_repo_base AS base LEFT JOIN repo_health_score_copy_ds AS hs ON base.repoUrl = hs.channel + LEFT JOIN project_insights_copy_repo_contributor_dependency AS cdep ON base.repoUrl = cdep.channel + LEFT JOIN project_insights_copy_repo_organization_dependency AS odep ON base.repoUrl = odep.channel LEFT JOIN project_insights_copy_repo_period_metrics AS rm ON base.repoUrl = rm.channel LEFT JOIN project_insights_copy_repo_star_deltas AS srd ON base.repoUrl = srd.repoUrl LEFT JOIN health_score_v2_repo_copy_ds AS hv2 ON base.repoUrl = hv2.repoUrl NODE project_insights_copy_results DESCRIPTION > - Union of project and repository insights + Union of project and repo insights; throws if a row type's dependency metrics are all zero SQL > - SELECT * - FROM project_insights_copy_project_results - UNION ALL - SELECT * - FROM project_insights_copy_repo_results + SELECT * EXCEPT (dependencyGuard) + FROM + ( + SELECT + *, + throwIf( + sum(contributorDependencyCount) OVER (PARTITION BY type) = 0 + OR sum(organizationDependencyCount) OVER (PARTITION BY type) = 0, + 'project_insights_copy: dependency metrics are all zero for a row type, check the health_score_*_dependency sources' + ) AS dependencyGuard + FROM + ( + SELECT * + FROM project_insights_copy_project_results + UNION ALL + SELECT * + FROM project_insights_copy_repo_results + ) + ) + WHERE dependencyGuard = 0 TYPE COPY TARGET_DATASOURCE project_insights_copy_ds diff --git a/services/libs/tinybird/pipes/projects_list.pipe b/services/libs/tinybird/pipes/projects_list.pipe index 9e6ad9712a..a80a127b8c 100644 --- a/services/libs/tinybird/pipes/projects_list.pipe +++ b/services/libs/tinybird/pipes/projects_list.pipe @@ -7,7 +7,7 @@ DESCRIPTION > - Parameters: - `search`: Optional string for searching projects (inherited from `insightsProjects_filtered`) - `count`: Optional boolean, when true returns project count instead of project data - - `orderByField`: Optional string specifying sort field, defaults to 'name'. Available fields: 'id', 'name', 'slug', 'description', 'logo', 'repositories', 'isLF', 'widgets', 'keywords', 'website', 'github', 'linkedin', 'twitter', 'score', 'rank', 'softwareValue', 'organizationCount', 'contributorCount', 'connectedPlatforms', 'firstCommit', 'repoData', 'healthScore' + - `orderByField`: Optional string specifying sort field, defaults to 'name'. Available fields: 'id', 'name', 'slug', 'description', 'logo', 'repositories', 'isLF', 'widgets', 'keywords', 'website', 'github', 'linkedin', 'twitter', 'score', 'rank', 'softwareValue', 'organizationCount', 'contributorCount', 'connectedPlatforms', 'firstCommit', 'repoData', 'healthScore', 'healthScoreV2', 'healthLabel', 'healthScoreV2Normalized', 'healthMaxScore' - `orderByDirection`: Optional string ('asc' or 'desc'), defaults to 'asc' - `pageSize`: Optional integer for result limit, defaults to 10 - `page`: Optional integer for pagination offset calculation, defaults to 0