fix: tag validator for plugins/datapacks (#7435)

* fix tag validator for plugins/datapacks

* Ignore some nags inside code blocks

* fix: tests

* fmt

---------

Co-authored-by: tdgao <mr.trumgao@gmail.com>

Author: aecsocket

Cargo.lock

@@ -154,7 +154,8 @@ postcard = { version = "1.1.3", default-features = false, features = ["alloc"] }
 postcard-bindgen = "0.8.0"
 proc-macro2 = { version = "1.0" }
 prometheus = "0.14.0"
-psl = "2.1.226"
+psl = "2.1.145"
+pulldown-cmark = { version = "0.13.4", default-features = false }
 quartz_nbt = "0.2.9"
 quick-xml = "0.38.3"
 quote = { version = "1.0" }

Cargo.toml

@@ -83,6 +83,7 @@ path-util = { workspace = true }
 postcard = { workspace = true }
 prometheus = { workspace = true }
 psl = { workspace = true }
+pulldown-cmark = { workspace = true }
 quick-xml = { workspace = true }
 rand = { workspace = true }
 rand_chacha = { workspace = true }

apps/labrinth/Cargo.toml

@@ -28,6 +28,20 @@ INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) V
 
 INSERT INTO loaders (id, loader, metadata) VALUES (7, 'bukkit', '{"platform":false}'::JSONB);
 INSERT INTO loaders (id, loader, metadata) VALUES (8, 'waterfall', '{"platform":true}'::JSONB);
+INSERT INTO loaders (id, loader) VALUES (9, 'datapack');
+
+INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id)
+SELECT l.id, pt.id
+FROM loaders l
+CROSS JOIN project_types pt
+WHERE
+	(l.loader IN ('bukkit', 'waterfall') AND pt.name = 'plugin')
+	OR (l.loader = 'datapack' AND pt.name = 'datapack');
+
+INSERT INTO loaders_project_types_games (loader_id, project_type_id, game_id)
+SELECT lpt.joining_loader_id, lpt.joining_project_type_id, 1
+FROM loaders_project_types lpt
+WHERE lpt.joining_loader_id IN (7, 8, 9);
 
 -- Adds dummies to mrpack_loaders
 INSERT INTO loader_field_enum_values (enum_id, value)

apps/labrinth/fixtures/dummy_data.sql

@@ -21,7 +21,7 @@ use super::{
 
 use super::{database::USER_USER_ID, get_json_val_str};
 
-pub const DUMMY_DATA_UPDATE: i64 = 8;
+pub const DUMMY_DATA_UPDATE: i64 = 9;
 
 pub const DUMMY_CATEGORIES: &[&str] = &[
     "combat",
@@ -47,6 +47,8 @@ pub enum TestFile {
     // and BasicModRandom.bytes() will return a different file each time.
     BasicModRandom { filename: String, bytes: Vec<u8> },
     BasicModpackRandom { filename: String, bytes: Vec<u8> },
+    BasicPluginRandom { filename: String, bytes: Vec<u8> },
+    BasicDatapackRandom { filename: String, bytes: Vec<u8> },
 }
 
 impl TestFile {
@@ -163,6 +165,59 @@ impl TestFile {
 
         TestFile::BasicModpackRandom { filename, bytes }
     }
+
+    pub fn build_random_plugin() -> Self {
+        let filename = format!("random-plugin-{}.jar", rand::random::<u64>());
+        let plugin_yml =
+            "name: TestPlugin\nversion: 1.0.0\nmain: com.example.TestPlugin\n";
+
+        let mut cursor = Cursor::new(Vec::new());
+        {
+            let mut zip = ZipWriter::new(&mut cursor);
+            zip.start_file(
+                "plugin.yml",
+                FileOptions::<()>::default()
+                    .compression_method(CompressionMethod::Stored),
+            )
+            .unwrap();
+            zip.write_all(plugin_yml.as_bytes()).unwrap();
+            zip.finish().unwrap();
+        }
+
+        TestFile::BasicPluginRandom {
+            filename,
+            bytes: cursor.into_inner(),
+        }
+    }
+
+    pub fn build_random_datapack() -> Self {
+        let filename = format!("random-datapack-{}.zip", rand::random::<u64>());
+        let pack_mcmeta = serde_json::json!({
+            "pack": {
+                "pack_format": 15,
+                "description": "Test datapack"
+            }
+        })
+        .to_string();
+
+        let mut cursor = Cursor::new(Vec::new());
+        {
+            let mut zip = ZipWriter::new(&mut cursor);
+            zip.start_file(
+                "pack.mcmeta",
+                FileOptions::<()>::default()
+                    .compression_method(CompressionMethod::Stored),
+            )
+            .unwrap();
+            zip.write_all(pack_mcmeta.as_bytes()).unwrap();
+            zip.finish().unwrap();
+        }
+
+        TestFile::BasicDatapackRandom {
+            filename,
+            bytes: cursor.into_inner(),
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -472,6 +527,8 @@ impl TestFile {
             TestFile::BasicModDifferent => "basic-mod-different.jar",
             TestFile::BasicModRandom { filename, .. } => filename,
             TestFile::BasicModpackRandom { filename, .. } => filename,
+            TestFile::BasicPluginRandom { filename, .. } => filename,
+            TestFile::BasicDatapackRandom { filename, .. } => filename,
         }
         .to_string()
     }
@@ -498,6 +555,8 @@ impl TestFile {
             }
             TestFile::BasicModRandom { bytes, .. } => bytes.clone(),
             TestFile::BasicModpackRandom { bytes, .. } => bytes.clone(),
+            TestFile::BasicPluginRandom { bytes, .. } => bytes.clone(),
+            TestFile::BasicDatapackRandom { bytes, .. } => bytes.clone(),
         }
     }
 
@@ -512,6 +571,8 @@ impl TestFile {
             TestFile::BasicZip => "resourcepack",
 
             TestFile::BasicModpackRandom { .. } => "modpack",
+            TestFile::BasicPluginRandom { .. } => "plugin",
+            TestFile::BasicDatapackRandom { .. } => "datapack",
         }
         .to_string()
     }
@@ -529,6 +590,10 @@ impl TestFile {
             TestFile::BasicModpackRandom { .. } => {
                 Some("application/x-modrinth-modpack+zip")
             }
+            TestFile::BasicPluginRandom { .. } => {
+                Some("application/java-archive")
+            }
+            TestFile::BasicDatapackRandom { .. } => Some("application/zip"),
         }
         .map(|s| s.to_string())
     }

apps/labrinth/src/test/dummy_data.rs

@@ -1,12 +1,14 @@
+mod markdown;
+
 use serde_json::json;
 
+use self::markdown::DescriptionMarkdown;
 use super::text::{
-    ProfanityKind, contains_description_spam, description_ends_with_header,
-    extract_description_blocks, extract_description_text,
-    find_banned_description_link, has_adjacent_same_level_headers,
+    ProfanityKind, contains_description_spam, extract_description_blocks,
+    extract_description_text, find_banned_description_link,
     has_image_without_alt_text, has_sufficient_english_blocks,
-    js_string_length, long_header_count, non_standard_text_ratio,
-    normalize_project_field_text, profanity_matches, project_requires_english,
+    js_string_length, non_standard_text_ratio, normalize_project_field_text,
+    profanity_matches, project_requires_english,
 };
 use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity};
 
@@ -19,11 +21,15 @@ const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05;
 pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
     let mut nags = Vec::new();
     let description = project.description.as_str();
+    let markdown = DescriptionMarkdown::parse(description);
+    let description_without_code = markdown.without_code();
     let normalized_description = normalize_project_field_text(description);
-    let text = extract_description_text(description);
-    let has_spam = has_description_spam(description);
-    let normalized_text = extract_description_text(&normalized_description);
-    let blocks = extract_description_blocks(description);
+    let text = extract_description_text(&description_without_code);
+    let has_spam = has_description_spam(&description_without_code);
+    let normalized_text =
+        normalize_project_field_text(&description_without_code);
+    let normalized_text = extract_description_text(&normalized_text);
+    let blocks = extract_description_blocks(&description_without_code);
     let profanity = profanity_matches(description);
 
     if let Some(matched) = profanity
@@ -51,7 +57,7 @@ pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
             .with_details(json!({ "value": matched.raw_text })),
         );
     }
-    if non_standard_text_ratio(description)
+    if non_standard_text_ratio(&description_without_code)
         >= NON_STANDARD_TEXT_FAILURE_THRESHOLD
     {
         nags.push(ProjectNag::new(
@@ -91,7 +97,7 @@ pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
             ProjectNagSeverity::Required,
         ));
     }
-    if let Some(url) = find_banned_description_link(description) {
+    if let Some(url) = find_banned_description_link(&description_without_code) {
         nags.push(
             ProjectNag::new(
                 ProjectNagKind::ProjectDescriptionBannedLink,
@@ -100,7 +106,7 @@ pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
             .with_details(json!({ "full_url": url })),
         );
     }
-    let long_headers = long_header_count(description);
+    let long_headers = markdown.long_header_count();
     if long_headers > 0 {
         nags.push(
             ProjectNag::new(
@@ -110,19 +116,19 @@ pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
             .with_details(json!({ "count": long_headers })),
         );
     }
-    if description_ends_with_header(description) {
+    if markdown.ends_with_header() {
         nags.push(ProjectNag::new(
             ProjectNagKind::DescriptionEndsWithHeader,
             ProjectNagSeverity::Required,
         ));
     }
-    if has_adjacent_same_level_headers(description) {
+    if markdown.has_adjacent_same_level_headers() {
         nags.push(ProjectNag::new(
             ProjectNagKind::AdjacentHeaders,
             ProjectNagSeverity::Required,
         ));
     }
-    if has_image_without_alt_text(description) {
+    if has_image_without_alt_text(&description_without_code) {
         nags.push(ProjectNag::new(
             ProjectNagKind::MissingAltText,
             ProjectNagSeverity::Warning,
@@ -133,8 +139,10 @@ pub(super) fn validate(project: &Project) -> Vec<ProjectNag> {
 }
 
 pub(super) fn is_non_english(project: &Project) -> bool {
-    let text = extract_description_text(&project.description);
-    let blocks = extract_description_blocks(&project.description);
+    let markdown = DescriptionMarkdown::parse(&project.description);
+    let description_without_code = markdown.without_code();
+    let text = extract_description_text(&description_without_code);
+    let blocks = extract_description_blocks(&description_without_code);
     is_non_english_text(project, &text, &blocks)
 }