From 98daf55348a968ac57b4893d9a4b211bb11082e3 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 18 Sep 2026 08:40:04 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat(pms):=20prompt=20profile=20=E6=9C=BA?= =?UTF-8?q?=E5=88=B6=20+=20=E8=BE=93=E5=87=BA=E9=97=A8/=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E5=B0=81=E9=A1=B6=E7=BB=84=E4=BB=B6=E5=8C=96=20+=20=E5=A4=8D?= =?UTF-8?q?=E4=BD=8D=E4=B8=A2=E5=A4=B1=E7=9A=84=E5=81=A5=E5=A3=AE=E6=80=A7?= =?UTF-8?q?=E8=A1=A5=E4=B8=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增版本化 prompt profile 加载器（结构 fail-closed 校验），10 个 generate method 全部支持 config 级模板覆盖 - pms_policy.v2 内置 profile：迁移 PMS 侧 v2 出题策略模板为正本 - 输出门注册表（pms_project_anchor）与 LLM 并发封顶成为 GenerateService 一等参数 - 复位历史上随 checkout 重建丢失的三个补丁：空响应重试、空抽取 prefill、logprob 调用关思考（env 开关语义不变） - 台账见 docs/PMS_PATCHES.md --- docs/PMS_PATCHES.md | 64 ++++++++++ graphgen/bases/base_generator.py | 40 +++++- .../extractor/schema_guided_extractor.py | 14 +++ .../models/generator/aggregated_generator.py | 14 +-- graphgen/models/generator/atomic_generator.py | 9 +- graphgen/models/generator/cot_generator.py | 13 +- .../generator/fill_in_blank_generator.py | 4 +- .../masked_fill_in_blank_generator.py | 9 +- .../generator/multi_answer_generator.py | 4 +- .../generator/multi_choice_generator.py | 4 +- .../models/generator/multi_hop_generator.py | 9 +- .../models/generator/true_false_generator.py | 4 +- graphgen/models/generator/vqa_generator.py | 8 +- graphgen/models/llm/api/openai_client.py | 70 +++++++++-- graphgen/operators/generate/gates.py | 53 ++++++++ .../operators/generate/generate_service.py | 32 +++++ .../operators/generate/llm_concurrency.py | 33 +++++ .../profiles/pms_policy.v2/atomic.json | 4 + .../profiles/pms_policy.v2/multi_hop.json | 4 + graphgen/templates/prompt_profiles.py | 119 ++++++++++++++++++ 20 files changed, 460 insertions(+), 51 deletions(-) create mode 100644 docs/PMS_PATCHES.md create mode 100644 graphgen/operators/generate/gates.py create mode 100644 graphgen/operators/generate/llm_concurrency.py create mode 100644 graphgen/templates/profiles/pms_policy.v2/atomic.json create mode 100644 graphgen/templates/profiles/pms_policy.v2/multi_hop.json create mode 100644 graphgen/templates/prompt_profiles.py diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md new file mode 100644 index 00000000..0d1ee4a9 --- /dev/null +++ b/docs/PMS_PATCHES.md @@ -0,0 +1,64 @@ +# PMS Fork 补丁台账 + +本仓库是 `open-sciencelab/GraphGen`（upstream，Apache-2.0）的自维护 fork，分支 +`pms/main`。上游修复可通过 rebase 同步，冲突以本 fork 语义为准。PMS 主仓库只钉 +住本 fork 的 commit SHA（receipt `_graphgen_source_identity` 自动记录）。 + +## #1 版本化 Prompt Profile（2026-09-18） + +- **动机**：官方所有 prompt 是 `graphgen/templates/` 下的硬编码常量，无任何 + config 覆盖机制；此前 PMS 侧只能 monkeypatch `generator.build_prompt`。 +- **实现**： + - `graphgen/templates/prompt_profiles.py`：profile 加载器。profile = 内置 id + （`graphgen/templates/profiles//`）或文件路径；每 method 一个 + `.json`；结构与官方模板逐键校验（fail-closed）。 + - `graphgen/bases/base_generator.py`：`apply_prompt_profile()` + + `template()`；generators 声明 `TEMPLATE_KEY` 并经 `self.template(...)` + 渲染；未应用 profile 时与官方常量逐字节一致。 + - 10 个 generate method（atomic/aggregated/multi_hop/cot/multi_choice/ + multi_answer/fill_in_blank/masked_fill_in_blank/true_false/vqa）全部接入。 + - `graphgen/operators/generate/generate_service.py`：generate 节点新增 + `prompt_profile` / `output_gate` / `llm_concurrency` 三个可选参数。 +- **内置 profile**：`pms_policy.v2`（atomic + multi_hop 的 PMS project-query + 出题策略模板，正本迁移自 PMS 受控侧 v2 模板）。 + +## #2 确定性输出门（2026-09-18） + +- `graphgen/operators/generate/gates.py`：注册表 + `pms_project_anchor` 门 + （题面必须含完整项目编号 P-/I-+≥6 位且答案非空，否则丢弃该 QA，不改写）。 +- generate 节点经 `output_gate` 参数启用。 + +## #3 生成 LLM 并发封顶（2026-09-18） + +- `graphgen/operators/generate/llm_concurrency.py`：事件循环级信号量， + 把同 loop 在途 generate 压到上限（35B-A3B 高并发实测会过载挂死）。 +- generate 节点经 `llm_concurrency` 参数启用（PMS 默认 4）。 + +## #4 空响应有界重试（复位自丢失补丁，2026-09-18） + +- `graphgen/models/llm/api/openai_client.py::generate_answer`：思考模型偶发 + 仅返回 `` 内容、过滤后为空串；`GRAPHGEN_EMPTY_RETRY=N`（默认 0， + 行为同官方）时对空响应做最多 N 次重试。历史版本此补丁仅存于 env 注入、 + 源码实现已丢失，本条为正确复位。 + +## #5 空抽取 assistant prefill 重试（复位自丢失补丁，2026-09-18） + +- `openai_client.generate_answer` 新增 `assistant_prefill` 参数； + `schema_guided_extractor.extract` 在响应为空且 + `GRAPHGEN_EMPTY_EXTRACTION_PREFILL=1` 时，以确定性 prefill `("entity"<|>` + 重试一次。非空响应永不重试/改写。 + +## #6 logprob 调用关思考（复位自丢失补丁，2026-09-18） + +- `generate_topk_per_token`：思考模型前若干 token 消耗在 `` 内，导致 + judge/loss 的 next-token yes/no logprob 退化。`GRAPHGEN_DISABLE_THINKING=1` + 时对 logprob 调用注入 `extra_body.chat_template_kwargs.enable_thinking=false` + （Qwen 兼容后端）。普通生成调用不受影响。 + +## 验收 + +- 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； +- PMS 主仓库 `tests/test_pms_graphgen_policy_generation.py`、 + `tests/test_run_graphgen_defaults.py` 为契约回归（rft 环境按路径加载 + 纯 stdlib 模块校验）； +- 真实生成由 GraphGen venv smoke 验收。 diff --git a/graphgen/bases/base_generator.py b/graphgen/bases/base_generator.py index 42e55180..04867b88 100644 --- a/graphgen/bases/base_generator.py +++ b/graphgen/bases/base_generator.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any +from typing import Any, Optional from graphgen.bases.base_llm_wrapper import BaseLLMWrapper @@ -7,10 +7,48 @@ class BaseGenerator(ABC): """ Generate QAs based on given prompts. + + PMS fork: prompt templates can be overridden per instance via a versioned + prompt profile (see graphgen.templates.prompt_profiles). Generators declare + their official template under ``TEMPLATE_KEY`` and render through + ``self.template(...)``; without a profile the official constant is used + byte-for-byte. """ + #: key into graphgen.templates.prompt_profiles.OFFICIAL_TEMPLATES + TEMPLATE_KEY: Optional[str] = None + def __init__(self, llm_client: BaseLLMWrapper): self.llm_client = llm_client + self._profile_templates: Optional[dict] = None + + def apply_prompt_profile(self, profile: str) -> None: + """Attach a versioned prompt profile to this generator instance.""" + from graphgen.templates.prompt_profiles import load_profile_template + + if self.TEMPLATE_KEY is None: + raise ValueError( + f"prompt_profile_unsupported_generator:{type(self).__name__}" + ) + self._profile_templates = load_profile_template(profile, self.TEMPLATE_KEY) + + def template(self, language: str, key: Optional[str] = None) -> str: + """Resolve the effective template for this generator. + + :param language: detected language key ("en" / "zh"). + :param key: nested sub-template key for multi-phase generators. + """ + from graphgen.templates.prompt_profiles import OFFICIAL_TEMPLATES + + if self.TEMPLATE_KEY is None: + raise ValueError( + f"prompt_template_key_missing:{type(self).__name__}" + ) + if self._profile_templates is not None: + node = self._profile_templates[language] + else: + node = OFFICIAL_TEMPLATES[self.TEMPLATE_KEY][language] + return node[key] if key is not None else node @staticmethod @abstractmethod diff --git a/graphgen/models/extractor/schema_guided_extractor.py b/graphgen/models/extractor/schema_guided_extractor.py index dee6e3e3..f594350f 100644 --- a/graphgen/models/extractor/schema_guided_extractor.py +++ b/graphgen/models/extractor/schema_guided_extractor.py @@ -59,10 +59,24 @@ def build_prompt(self, text: str) -> str: return prompt async def extract(self, chunk: Chunk) -> dict: + import os + text = chunk.content prompt = self.build_prompt(text) response = await self.llm_client.generate_answer(prompt) + + # PMS fork (docs/PMS_PATCHES.md #5): strict tuple extraction prompts + # occasionally hit EOS as the first token. When enabled and the first + # response is empty, retry once with a deterministic assistant prefill + # that seeds the opening of the expected output; non-empty responses + # are never retried or modified. + if not response.strip() and os.environ.get("GRAPHGEN_EMPTY_EXTRACTION_PREFILL", "0") == "1": + prefill = '("entity"<|>' + logger.warning("Empty extraction response; retrying with assistant prefill") + response = await self.llm_client.generate_answer( + prompt, assistant_prefill=prefill + ) try: extracted_info = json.loads(response) # Ensure all required keys are present diff --git a/graphgen/models/generator/aggregated_generator.py b/graphgen/models/generator/aggregated_generator.py index 3f223325..c8ef06c7 100644 --- a/graphgen/models/generator/aggregated_generator.py +++ b/graphgen/models/generator/aggregated_generator.py @@ -2,7 +2,6 @@ from typing import Any, Optional from graphgen.bases import BaseGenerator -from graphgen.templates import AGGREGATED_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger @@ -14,9 +13,11 @@ class AggregatedGenerator(BaseGenerator): 2. question generation: Generate relevant questions based on the rephrased text. """ - @staticmethod + TEMPLATE_KEY = "aggregated" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for REPHRASE. @@ -51,7 +52,7 @@ def build_prompt( # for index, text in enumerate(original_text) # ] # ) - prompt = AGGREGATED_GENERATION_PROMPT[language]["ANSWER_REPHRASING"].format( + prompt = self.template(language, "ANSWER_REPHRASING").format( entities=entities_str, relationships=relations_str ) return prompt @@ -73,15 +74,14 @@ def parse_rephrased_text(response: str) -> Optional[str]: return None return rephrased_text.strip('"').strip("'") - @staticmethod - def _build_prompt_for_question_generation(answer: str) -> str: + def _build_prompt_for_question_generation(self, answer: str) -> str: """ Build prompts for QUESTION GENERATION. :param answer: :return: """ language = detect_main_language(answer) - prompt = AGGREGATED_GENERATION_PROMPT[language]["QUESTION_GENERATION"].format( + prompt = self.template(language, "QUESTION_GENERATION").format( answer=answer ) return prompt diff --git a/graphgen/models/generator/atomic_generator.py b/graphgen/models/generator/atomic_generator.py index d045b0da..a77b731c 100644 --- a/graphgen/models/generator/atomic_generator.py +++ b/graphgen/models/generator/atomic_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import ATOMIC_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class AtomicGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "atomic" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch context = "" @@ -19,7 +20,7 @@ def build_prompt( context += f"- {edge[0]} - {edge[1]}: {edge[2]['description']}\n" language = detect_main_language(context) - prompt = ATOMIC_GENERATION_PROMPT[language].format(context=context) + prompt = self.template(language).format(context=context) return prompt @staticmethod diff --git a/graphgen/models/generator/cot_generator.py b/graphgen/models/generator/cot_generator.py index 88d04324..b4edd38c 100644 --- a/graphgen/models/generator/cot_generator.py +++ b/graphgen/models/generator/cot_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import COT_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class CoTGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "cot" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for COT Template Design. @@ -30,13 +31,13 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = COT_GENERATION_PROMPT[language]["COT_TEMPLATE_DESIGN"].format( + prompt = self.template(language, "COT_TEMPLATE_DESIGN").format( entities=entities_str, relationships=relationships_str ) return prompt - @staticmethod def build_prompt_for_cot_generation( + self, batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], question: str, reasoning_path: str, @@ -58,7 +59,7 @@ def build_prompt_for_cot_generation( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = COT_GENERATION_PROMPT[language]["COT_GENERATION"].format( + prompt = self.template(language, "COT_GENERATION").format( entities=entities_str, relationships=relationships_str, question=question, diff --git a/graphgen/models/generator/fill_in_blank_generator.py b/graphgen/models/generator/fill_in_blank_generator.py index a26daf3e..2d63d8d1 100644 --- a/graphgen/models/generator/fill_in_blank_generator.py +++ b/graphgen/models/generator/fill_in_blank_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import FILL_IN_BLANK_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class FillInBlankGenerator(BaseGenerator): + TEMPLATE_KEY = "fill_in_blank" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -92,7 +92,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = FILL_IN_BLANK_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/masked_fill_in_blank_generator.py b/graphgen/models/generator/masked_fill_in_blank_generator.py index 355cadcf..ab941688 100644 --- a/graphgen/models/generator/masked_fill_in_blank_generator.py +++ b/graphgen/models/generator/masked_fill_in_blank_generator.py @@ -3,7 +3,6 @@ from typing import Any, Optional from graphgen.bases import BaseGenerator -from graphgen.templates import AGGREGATED_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger random.seed(42) @@ -16,9 +15,11 @@ class MaskedFillInBlankGenerator(BaseGenerator): 2. mask: Randomly select a node from the input nodes, and then mask the name of the node in the rephrased text. """ - @staticmethod + TEMPLATE_KEY = "masked_fill_in_blank" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for REPHRASE. @@ -53,7 +54,7 @@ def build_prompt( # for index, text in enumerate(original_text) # ] # ) - prompt = AGGREGATED_GENERATION_PROMPT[language]["ANSWER_REPHRASING"].format( + prompt = self.template(language, "ANSWER_REPHRASING").format( entities=entities_str, relationships=relations_str ) return prompt diff --git a/graphgen/models/generator/multi_answer_generator.py b/graphgen/models/generator/multi_answer_generator.py index a341a4fd..c9af5701 100644 --- a/graphgen/models/generator/multi_answer_generator.py +++ b/graphgen/models/generator/multi_answer_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MAQ_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiAnswerGenerator(BaseGenerator): + TEMPLATE_KEY = "multi_answer" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -113,7 +113,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = MAQ_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/multi_choice_generator.py b/graphgen/models/generator/multi_choice_generator.py index 0c48b76d..3a16a550 100644 --- a/graphgen/models/generator/multi_choice_generator.py +++ b/graphgen/models/generator/multi_choice_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MCQ_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiChoiceGenerator(BaseGenerator): + TEMPLATE_KEY = "multi_choice" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -111,7 +111,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = MCQ_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/multi_hop_generator.py b/graphgen/models/generator/multi_hop_generator.py index a19082b9..232f0e56 100644 --- a/graphgen/models/generator/multi_hop_generator.py +++ b/graphgen/models/generator/multi_hop_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MULTI_HOP_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiHopGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "multi_hop" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch entities_str = "\n".join( @@ -26,7 +27,7 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = MULTI_HOP_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( entities=entities_str, relationships=relationships_str ) return prompt diff --git a/graphgen/models/generator/true_false_generator.py b/graphgen/models/generator/true_false_generator.py index 1a1fa0d3..c51fe371 100644 --- a/graphgen/models/generator/true_false_generator.py +++ b/graphgen/models/generator/true_false_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import TF_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class TrueFalseGenerator(BaseGenerator): + TEMPLATE_KEY = "true_false" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -84,7 +84,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = TF_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/vqa_generator.py b/graphgen/models/generator/vqa_generator.py index 723bd2a6..a6375a10 100644 --- a/graphgen/models/generator/vqa_generator.py +++ b/graphgen/models/generator/vqa_generator.py @@ -3,14 +3,14 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import VQA_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class VQAGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "vqa" def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch entities_str = "\n".join( @@ -27,7 +27,7 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = VQA_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( entities=entities_str, relationships=relationships_str ) return prompt diff --git a/graphgen/models/llm/api/openai_client.py b/graphgen/models/llm/api/openai_client.py index 532b981c..916424af 100644 --- a/graphgen/models/llm/api/openai_client.py +++ b/graphgen/models/llm/api/openai_client.py @@ -13,6 +13,7 @@ from graphgen.bases.base_llm_wrapper import BaseLLMWrapper from graphgen.bases.datatypes import Token from graphgen.models.llm.limitter import RPM, TPM +from graphgen.utils import logger def get_top_response_tokens(response: openai.ChatCompletion) -> List[Token]: @@ -86,7 +87,9 @@ def __post_init__(self): else: raise ValueError(f"Unsupported backend {self.backend}. Use 'openai_api' or 'azure_openai_api'.") - def _pre_generate(self, text: str, history: List[str]) -> Dict: + def _pre_generate( + self, text: str, history: List[str], assistant_prefill: Optional[str] = None + ) -> Dict: kwargs = { "temperature": self.temperature, "top_p": self.top_p, @@ -106,6 +109,11 @@ def _pre_generate(self, text: str, history: List[str]) -> Dict: assert len(history) % 2 == 0, "History should have even number of elements." messages = history + messages + # PMS fork (docs/PMS_PATCHES.md #5): deterministic assistant prefill for + # strict-prompt backends that occasionally emit EOS as the first token. + if assistant_prefill is not None: + messages.append({"role": "assistant", "content": assistant_prefill}) + kwargs["messages"] = messages return kwargs @@ -130,6 +138,18 @@ async def generate_topk_per_token( # Limit max_tokens to 1 to avoid long completions kwargs["max_tokens"] = 1 + # PMS fork (docs/PMS_PATCHES.md #6): thinking models spend the first + # tokens inside content, which makes next-token yes/no logprobs + # meaningless for judge/loss calls. GRAPHGEN_DISABLE_THINKING=1 asks the + # backend to disable thinking for these logprob-based calls only. + import os + + if os.environ.get("GRAPHGEN_DISABLE_THINKING", "0") == "1": + kwargs["extra_body"] = { + **kwargs.get("extra_body", {}), + "chat_template_kwargs": {"enable_thinking": False}, + } + completion = await self.client.chat.completions.create( # pylint: disable=E1125 model=self.model, **kwargs ) @@ -149,9 +169,10 @@ async def generate_answer( self, text: str, history: Optional[List[str]] = None, + assistant_prefill: Optional[str] = None, **extra: Any, ) -> str: - kwargs = self._pre_generate(text, history) + kwargs = self._pre_generate(text, history, assistant_prefill) prompt_tokens = 0 for message in kwargs["messages"]: @@ -162,18 +183,41 @@ async def generate_answer( await self.rpm.wait(silent=True) await self.tpm.wait(estimated_tokens, silent=True) - completion = await self.client.chat.completions.create( # pylint: disable=E1125 - model=self.model, **kwargs - ) - if hasattr(completion, "usage"): - self.token_usage.append( - { - "prompt_tokens": completion.usage.prompt_tokens, - "completion_tokens": completion.usage.completion_tokens, - "total_tokens": completion.usage.total_tokens, - } + # PMS fork (docs/PMS_PATCHES.md #4): bounded retry on empty responses. + # Thinking backends occasionally return only content that is + # stripped to nothing downstream; retry instead of propagating an empty + # string that would fail parsing. GRAPHGEN_EMPTY_RETRY=0 keeps official + # single-attempt behavior. + import os + + try: + empty_retries = max(0, int(os.environ.get("GRAPHGEN_EMPTY_RETRY", "0"))) + except ValueError: + empty_retries = 0 + + content = "" + for attempt in range(empty_retries + 1): + completion = await self.client.chat.completions.create( # pylint: disable=E1125 + model=self.model, **kwargs + ) + if hasattr(completion, "usage"): + self.token_usage.append( + { + "prompt_tokens": completion.usage.prompt_tokens, + "completion_tokens": completion.usage.completion_tokens, + "total_tokens": completion.usage.total_tokens, + } + ) + content = self.filter_think_tags(completion.choices[0].message.content) + if content.strip(): + break + logger.warning( + "Empty LLM response (attempt %d/%d, model=%s)", + attempt + 1, + empty_retries + 1, + self.model, ) - return self.filter_think_tags(completion.choices[0].message.content) + return content async def generate_inputs_prob( self, text: str, history: Optional[List[str]] = None, **extra: Any diff --git a/graphgen/operators/generate/gates.py b/graphgen/operators/generate/gates.py new file mode 100644 index 00000000..63eb254a --- /dev/null +++ b/graphgen/operators/generate/gates.py @@ -0,0 +1,53 @@ +"""Deterministic output gates for generated QA pairs (PMS fork capability). + +A gate is a pure, synchronous filter applied to the parsed QA list of one +generate call before results reach the native cache. Gates never modify or +fabricate model output; they only drop pairs that violate a declared +deterministic contract. See docs/PMS_PATCHES.md #2. +""" + +from __future__ import annotations + +import re +from typing import Any, Callable + +GATES: dict[str, Callable[[Any], list[dict]]] = {} + +_PROJECT_ID_PATTERN = re.compile(r"(?:P|I)-\d{6,}") + + +def register_gate(name: str) -> Callable: + def _decorator(func: Callable[[Any], list[dict]]) -> Callable[[Any], list[dict]]: + if name in GATES: + raise ValueError(f"output_gate_already_registered:{name}") + GATES[name] = func + return func + return _decorator + + +@register_gate("pms_project_anchor") +def pms_project_anchor(qa_pairs: Any) -> list[dict]: + """PMS project-query anchor gate. + + Only pairs whose question carries a complete project id (P-/I- plus at + least 6 digits) and a non-empty answer are executable by the project-query + skill; name-only or refusal placeholder questions are dropped. + """ + if not isinstance(qa_pairs, list): + return [] + return [ + pair + for pair in qa_pairs + if isinstance(pair, dict) + and isinstance(pair.get("question"), str) + and bool(_PROJECT_ID_PATTERN.search(pair["question"])) + and isinstance(pair.get("answer"), str) + and pair["answer"].strip() + ] + + +def apply_output_gate(name: str, qa_pairs: Any) -> list[dict]: + """Apply a registered gate by name; fail-closed on unknown gates.""" + if name not in GATES: + raise ValueError(f"output_gate_not_registered:{name}") + return GATES[name](qa_pairs) diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 18ce1d43..52359b29 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -9,6 +9,13 @@ class GenerateService(BaseOperator): """ Generate question-answer pairs based on nodes and edges. + + PMS fork additions (all optional, default = official behavior): + - ``prompt_profile``: versioned prompt override loaded via + graphgen.templates.prompt_profiles; + - ``output_gate``: deterministic QA filter registered in + graphgen.operators.generate.gates; + - ``llm_concurrency``: cap on in-flight LLM requests per event loop. """ def __init__( @@ -17,6 +24,9 @@ def __init__( kv_backend: str = "rocksdb", method: str = "aggregated", data_format: str = "ChatML", + prompt_profile: str | None = None, + output_gate: str | None = None, + llm_concurrency: int | None = None, **generate_kwargs, ): super().__init__( @@ -85,6 +95,28 @@ def __init__( else: raise ValueError(f"Unsupported generation mode: {method}") + if prompt_profile: + self.generator.apply_prompt_profile(prompt_profile) + if llm_concurrency is not None: + from graphgen.operators.generate.llm_concurrency import ( + bound_llm_concurrency, + ) + + limited = bound_llm_concurrency(self.generator.generate, llm_concurrency) + self.generator.generate = limited + if output_gate: + from graphgen.operators.generate.gates import GATES + + if output_gate not in GATES: + raise ValueError(f"output_gate_not_registered:{output_gate}") + gate = GATES[output_gate] + base_generate = self.generator.generate + + async def _gated_generate(batch: list) -> list[dict]: + return gate(await base_generate(batch)) + + self.generator.generate = _gated_generate + def process(self, batch: list) -> Tuple[list, dict]: """ Generate question-answer pairs based on nodes and edges. diff --git a/graphgen/operators/generate/llm_concurrency.py b/graphgen/operators/generate/llm_concurrency.py new file mode 100644 index 00000000..685809e6 --- /dev/null +++ b/graphgen/operators/generate/llm_concurrency.py @@ -0,0 +1,33 @@ +"""Bound in-flight LLM requests of a generator inside one event loop. + +Official ``run_concurrent`` has no cross-batch concurrency cap; a busy LLM +endpoint (e.g. a 35B-A3B deployment) can deadlock under high fan-out. This +helper caps concurrent ``generator.generate`` calls with an asyncio.Semaphore +created lazily per running loop (Ray actors create a fresh loop per batch). +Pure stdlib so PMS-side contract tests can import this module by path without +the graphgen package installed. See docs/PMS_PATCHES.md #3. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + + +def bound_llm_concurrency(generate: Any, limit: int) -> Any: + """Wrap an async ``generate(batch)`` so at most ``limit`` run in-flight.""" + if limit < 1: + raise ValueError("llm_concurrency_must_be_positive") + state: dict[str, Any] = {} + + async def _gated_generate(batch: Any) -> Any: + loop = asyncio.get_running_loop() + semaphore = state.get("semaphore") + if semaphore is None or state.get("loop") is not loop: + semaphore = asyncio.Semaphore(limit) + state["semaphore"] = semaphore + state["loop"] = loop + async with semaphore: + return await generate(batch) + + return _gated_generate diff --git a/graphgen/templates/profiles/pms_policy.v2/atomic.json b/graphgen/templates/profiles/pms_policy.v2/atomic.json new file mode 100644 index 00000000..ff6b8411 --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v2/atomic.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定图谱片段：\n{context}\n\n输出：\n" +} diff --git a/graphgen/templates/profiles/pms_policy.v2/multi_hop.json b/graphgen/templates/profiles/pms_policy.v2/multi_hop.json new file mode 100644 index 00000000..bd6798a0 --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v2/multi_hop.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" +} diff --git a/graphgen/templates/prompt_profiles.py b/graphgen/templates/prompt_profiles.py new file mode 100644 index 00000000..a825a674 --- /dev/null +++ b/graphgen/templates/prompt_profiles.py @@ -0,0 +1,119 @@ +"""Versioned prompt profiles: config-driven prompt customization for generators. + +PMS fork capability (see docs/PMS_PATCHES.md #1). Official GraphGen hard-codes all +prompts as Python constants under ``graphgen.templates``; this module adds a +generic, fail-closed override channel: + +- a *profile* is a directory containing one JSON file per generator + ``TEMPLATE_KEY`` (e.g. ``atomic.json``); +- each JSON file must mirror the exact key structure of the corresponding + official template (same languages, same nested keys, leaf values are str); +- ``profile`` values are either built-in profile ids resolved against + ``graphgen/templates/profiles/`` or explicit filesystem paths (containing + a path separator). + +Default behaviour is untouched: when no profile is applied every generator +falls back to the official template constant byte-for-byte. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from graphgen.templates import ( + AGGREGATED_GENERATION_PROMPT, + ATOMIC_GENERATION_PROMPT, + COT_GENERATION_PROMPT, + FILL_IN_BLANK_GENERATION_PROMPT, + MAQ_GENERATION_PROMPT, + MCQ_GENERATION_PROMPT, + MULTI_HOP_GENERATION_PROMPT, + TF_GENERATION_PROMPT, + VQA_GENERATION_PROMPT, +) + +BUILTIN_PROFILE_ROOT = Path(__file__).resolve().parent / "profiles" + +#: template_key -> official template constant (structure validation source). +OFFICIAL_TEMPLATES: dict[str, Any] = { + "atomic": ATOMIC_GENERATION_PROMPT, + "multi_hop": MULTI_HOP_GENERATION_PROMPT, + "aggregated": AGGREGATED_GENERATION_PROMPT, + "cot": COT_GENERATION_PROMPT, + "multi_choice": MCQ_GENERATION_PROMPT, + "multi_answer": MAQ_GENERATION_PROMPT, + "fill_in_blank": FILL_IN_BLANK_GENERATION_PROMPT, + "masked_fill_in_blank": AGGREGATED_GENERATION_PROMPT, + "true_false": TF_GENERATION_PROMPT, + "vqa": VQA_GENERATION_PROMPT, +} + + +def resolve_profile_dir(profile: str) -> Path: + """Resolve a profile id / path to its directory; fail-closed if missing.""" + if not profile or not isinstance(profile, str): + raise ValueError("prompt_profile_must_be_non_empty_string") + if "/" in profile or "\\" in profile: + profile_dir = Path(profile) + else: + profile_dir = BUILTIN_PROFILE_ROOT / profile + if not profile_dir.is_dir(): + raise ValueError(f"prompt_profile_not_found:{profile}") + return profile_dir + + +def _validate_structure(official: Any, override: Any, path: str) -> None: + """Override must mirror the official template's key structure exactly.""" + if isinstance(official, dict): + if not isinstance(override, dict): + raise ValueError(f"prompt_profile_structure_mismatch:{path}") + if set(override.keys()) != set(official.keys()): + raise ValueError( + f"prompt_profile_structure_mismatch:{path}:" + f"keys={sorted(map(str, override.keys()))}" + ) + for key in official: + _validate_structure(official[key], override[key], f"{path}.{key}") + return + if not isinstance(override, str) or not override.strip(): + raise ValueError(f"prompt_profile_structure_mismatch:{path}:non_string_leaf") + + +def load_profile_template(profile: str, template_key: str) -> dict[str, Any]: + """Load one generator's template override from a profile. + + :param profile: profile id (builtin) or filesystem path. + :param template_key: generator TEMPLATE_KEY, must exist in OFFICIAL_TEMPLATES. + :return: parsed override dict (same structure as the official template). + """ + if template_key not in OFFICIAL_TEMPLATES: + raise ValueError(f"prompt_profile_unknown_template_key:{template_key}") + profile_file = resolve_profile_dir(profile) / f"{template_key}.json" + if not profile_file.is_file(): + raise ValueError(f"prompt_profile_template_missing:{profile}/{template_key}") + try: + override = json.loads(profile_file.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise ValueError(f"prompt_profile_json_invalid:{profile_file}:{error}") from error + _validate_structure(OFFICIAL_TEMPLATES[template_key], override, template_key) + return override + + +def profile_template_sha256s(profile: str, template_keys: list[str]) -> dict[str, str]: + """Lineage fingerprint: SHA-256 of each override file actually in use.""" + profile_dir = resolve_profile_dir(profile) + digests: dict[str, str] = {} + for template_key in template_keys: + profile_file = profile_dir / f"{template_key}.json" + if not profile_file.is_file(): + continue + digests[template_key] = hashlib.sha256(profile_file.read_bytes()).hexdigest() + return digests + + +def apply_prompt_profile(generator: Any, profile: str) -> None: + """Attach profile templates to a generator instance (BaseGenerator API).""" + generator.apply_prompt_profile(profile) From 2adbbe4c52f9776dc3538fc3f954bb19707cb0f5 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 9 Oct 2026 09:26:27 +0000 Subject: [PATCH 2/4] =?UTF-8?q?feat(pms):=20Phase2=20=E4=BA=8B=E4=BB=B6-?= =?UTF-8?q?=E5=AE=9E=E4=BD=93=E6=98=9F=E5=9E=8B=E5=9B=BE=E7=AE=A1=E7=BA=BF?= =?UTF-8?q?=EF=BC=88SAG=20=E7=A7=BB=E6=A4=8D=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 事件抽取：SAG extract_document v3.1 提示词改编 + 严格响应合同 + 数字接地校验（MIT 归因） - 星型编码进图：事件=EVENT 节点+成员边（边属性含角色），merge/SEP 血缘复用官方 LightRAG - build_kg kg_method 分发：event_entity 与 light_rag 并存，默认行为逐字节不变 - event_join 分区器：共享实体 join + 同锚硬约束，产出官方 Community，metadata 带 event_ids - 审计链：生成端引用分区节点集合校验，失败剥离保留题面；pms_policy.v3 profile - 台账 docs/PMS_PATCHES.md #7；fork 测试基线与上游一致（23F/6P，零回归） Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 18 +++ graphgen/bases/base_generator.py | 7 +- graphgen/models/__init__.py | 2 + graphgen/models/generator/atomic_generator.py | 7 +- .../models/generator/multi_hop_generator.py | 7 +- graphgen/models/generator/support.py | 42 ++++++ graphgen/models/kg_builder/event_contract.py | 119 +++++++++++++++ .../kg_builder/event_entity_kg_builder.py | 135 ++++++++++++++++++ graphgen/models/partitioner/__init__.py | 1 + .../partitioner/event_join_partitioner.py | 113 +++++++++++++++ graphgen/operators/build_kg/build_event_kg.py | 51 +++++++ .../operators/build_kg/build_kg_service.py | 13 ++ .../operators/generate/generate_service.py | 15 +- .../operators/partition/partition_service.py | 8 ++ graphgen/templates/__init__.py | 7 +- graphgen/templates/kg/__init__.py | 1 + .../templates/kg/event_entity_extraction.py | 113 +++++++++++++++ .../profiles/pms_policy.v3/atomic.json | 4 + .../profiles/pms_policy.v3/multi_hop.json | 4 + 19 files changed, 659 insertions(+), 8 deletions(-) create mode 100644 graphgen/models/generator/support.py create mode 100644 graphgen/models/kg_builder/event_contract.py create mode 100644 graphgen/models/kg_builder/event_entity_kg_builder.py create mode 100644 graphgen/models/partitioner/event_join_partitioner.py create mode 100644 graphgen/operators/build_kg/build_event_kg.py create mode 100644 graphgen/templates/kg/event_entity_extraction.py create mode 100644 graphgen/templates/profiles/pms_policy.v3/atomic.json create mode 100644 graphgen/templates/profiles/pms_policy.v3/multi_hop.json diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 0d1ee4a9..88d2fb8e 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -55,6 +55,24 @@ 时对 logprob 调用注入 `extra_body.chat_template_kwargs.enable_thinking=false` （Qwen 兼容后端）。普通生成调用不受影响。 +## #7 事件-实体星型图管线（Phase 2，2026-10-09） + +SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_reference/`，方案见 PMS 主仓 +`docs/2026-09-18-graphgen-phase2-sag-event-entity-plan.md` §3.2 复用地图）： + +- `templates/kg/event_entity_extraction.py`：事件抽取提示词（SAG extract_document v3.1 改编 + + PMS 锚点/字段锁定/数字接地铁律）。 +- `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 + 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 +- `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， + 边属性含角色）；合并/摘要/SEP 血缘复用官方 LightRAG merge。 +- `operators/build_kg/`：`kg_method: event_entity` 分发（默认 light_rag 与官方逐字节一致）。 +- `models/partitioner/event_join_partitioner.py`：共享实体 join + 同锚硬约束的事件簇分区 + （SAG join 语义离线物化），产出官方 `Community`，metadata 携带 event_ids。 +- `models/generator/support.py` + atomic/multi_hop 解析 + GenerateService 校验： + `{"cited": [...]}` 引用必须存在于本分区节点集合，失败剥离保留题面。 +- `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 + ## 验收 - 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； diff --git a/graphgen/bases/base_generator.py b/graphgen/bases/base_generator.py index 04867b88..9307dea0 100644 --- a/graphgen/bases/base_generator.py +++ b/graphgen/bases/base_generator.py @@ -84,6 +84,8 @@ def format_generation_results( ) -> dict[str, Any]: question = result.get("question", "") answer = result.get("answer", "") + # PMS fork（Phase 2）：已校验的 引用块随 QA_pairs 输出透传。 + support = result.get("support") if "options" in result and result["options"]: options = result["options"] options_str = "\n".join( @@ -114,8 +116,11 @@ def format_generation_results( } if output_data_format == "QA_pairs": - return { + output = { "question": question, "answer": answer, } + if support: + output["support"] = support + return output raise ValueError(f"Unknown output data format: {output_data_format}") diff --git a/graphgen/models/__init__.py b/graphgen/models/__init__.py index 8bd2c9d8..d6c9306e 100644 --- a/graphgen/models/__init__.py +++ b/graphgen/models/__init__.py @@ -30,6 +30,7 @@ BFSPartitioner, DFSPartitioner, ECEPartitioner, + EventJoinPartitioner, LeidenPartitioner, QuintuplePartitioner, TriplePartitioner, @@ -89,6 +90,7 @@ "BFSPartitioner": ".partitioner", "DFSPartitioner": ".partitioner", "ECEPartitioner": ".partitioner", + "EventJoinPartitioner": ".partitioner", "LeidenPartitioner": ".partitioner", "TriplePartitioner": ".partitioner", "QuintuplePartitioner": ".partitioner", diff --git a/graphgen/models/generator/atomic_generator.py b/graphgen/models/generator/atomic_generator.py index a77b731c..38fe9d98 100644 --- a/graphgen/models/generator/atomic_generator.py +++ b/graphgen/models/generator/atomic_generator.py @@ -2,6 +2,7 @@ from typing import Any from graphgen.bases import BaseGenerator +from graphgen.models.generator.support import parse_support from graphgen.utils import detect_main_language, logger @@ -45,4 +46,8 @@ def parse_response(response: str) -> list[dict]: answer = answer.strip('"').strip("'") logger.debug("Question: %s", question) logger.debug("Answer: %s", answer) - return [{"question": question, "answer": answer}] + qa_pairs = [{"question": question, "answer": answer}] + support = parse_support(response) + if support: + qa_pairs[0]["support"] = support + return qa_pairs diff --git a/graphgen/models/generator/multi_hop_generator.py b/graphgen/models/generator/multi_hop_generator.py index 232f0e56..da0a4615 100644 --- a/graphgen/models/generator/multi_hop_generator.py +++ b/graphgen/models/generator/multi_hop_generator.py @@ -2,6 +2,7 @@ from typing import Any from graphgen.bases import BaseGenerator +from graphgen.models.generator.support import parse_support from graphgen.utils import detect_main_language, logger @@ -48,4 +49,8 @@ def parse_response(response: str) -> list[dict]: answer = answer.strip('"').strip("'") logger.debug("Question: %s", question) logger.debug("Answer: %s", answer) - return [{"question": question, "answer": answer}] + qa_pairs = [{"question": question, "answer": answer}] + support = parse_support(response) + if support: + qa_pairs[0]["support"] = support + return qa_pairs diff --git a/graphgen/models/generator/support.py b/graphgen/models/generator/support.py new file mode 100644 index 00000000..810f65d2 --- /dev/null +++ b/graphgen/models/generator/support.py @@ -0,0 +1,42 @@ +""" 审计块解析（PMS fork，Phase 2 升级 C）。 + +生成器可在 / 之后输出 +``{"cited": ["节点或事件名称", ...]}``； +本模块只负责解析，有效性校验（cited 是否真存在于出题分区）在 +GenerateService.process 中按分区的节点集合执行，失败剥离 support 保留题面。 +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Optional + +_SUPPORT_RE = re.compile(r"(.*?)", re.DOTALL) + + +def parse_support(response: str) -> Optional[dict[str, Any]]: + """从模型响应解析 support 块；缺失/畸形返回 None（不影响题面解析）。""" + match = _SUPPORT_RE.search(response or "") + if not match: + return None + try: + payload = json.loads(match.group(1).strip()) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict) or set(payload.keys()) != {"cited"}: + return None + cited = payload.get("cited") + if not isinstance(cited, list) or not all(isinstance(c, str) and c.strip() for c in cited): + return None + return {"cited": [c.strip() for c in cited]} + + +def validate_support(support: Optional[dict[str, Any]], node_names: set[str]) -> Optional[dict[str, Any]]: + """cited 全部存在于分区节点集合 → 原样返回；否则返回 None（剥离）。""" + if not support: + return None + cited = support.get("cited") or [] + if cited and set(cited) <= node_names: + return support + return None diff --git a/graphgen/models/kg_builder/event_contract.py b/graphgen/models/kg_builder/event_contract.py new file mode 100644 index 00000000..374ea557 --- /dev/null +++ b/graphgen/models/kg_builder/event_contract.py @@ -0,0 +1,119 @@ +"""Pure stdlib contracts for SAG-adapted event extraction (testable without GraphGen).""" +from __future__ import annotations +import hashlib +import json +import re +from collections import defaultdict +from typing import Any, Optional + +PMS_ENTITY_TYPES: list[dict] = [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"}, +] + +PROJECT_ID_RE = re.compile(r"\b[PI]-\d{6,}\b") +_NUMBER_RE = re.compile(r"(? set[str]: + """原文/事件中的数字集合（SAG grounding 的确定性口径）。""" + return set(_NUMBER_RE.findall(text or "")) + + +def event_anchor(content: str) -> str | None: + """事件的完整项目编号锚点；取首个出现的编号。""" + match = PROJECT_ID_RE.search(content or "") + return match.group(0) if match else None + + +def event_node_name(anchor: str, title: str, content: str) -> str: + """确定性事件节点 id：event:{anchor}:{hash12}（内容 SHA，重叠 chunk 天然去重）。""" + digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] + return f"event:{anchor}:{digest}" + + +def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[list[dict], dict[str, int]]: + """解析并校验 LLM 事件抽取响应，返回 (有效事件列表, 丢弃计数)。 + + 严格合同（SAG schema.py 口径）：顶层/事件/实体禁合同外字段；is_valid=False、 + 无项目锚点、数字不接地、实体类型越界的事件整条丢弃并计数。 + """ + dropped: dict[str, int] = defaultdict(int) + text = (response or "").strip() + if text.startswith("```"): + text = text.strip("`") + if text.startswith("json"): + text = text[4:] + text = text.strip() + try: + payload = json.loads(text) + except json.JSONDecodeError: + dropped["response_not_json"] += 1 + return [], dict(dropped) + if not isinstance(payload, dict) or set(payload.keys()) != {"type", "data"}: + dropped["response_contract_invalid"] += 1 + return [], dict(dropped) + data = payload.get("data") + if not isinstance(data, dict) or set(data.keys()) != {"items"}: + dropped["response_contract_invalid"] += 1 + return [], dict(dropped) + + evidence_numbers = numbers_in(chunk_content) + valid_events: list[dict] = [] + for item in data.get("items") or []: + if not isinstance(item, dict): + dropped["event_not_object"] += 1 + continue + extra_keys = set(item.keys()) - _ALLOWED_EVENT_KEYS + if extra_keys: + dropped["event_extra_fields"] += 1 + continue + title = str(item.get("title") or "").strip() + content = str(item.get("content") or "").strip() + entities_raw = item.get("entities") or [] + if not title or not content: + dropped["event_blank"] += 1 + continue + if item.get("is_valid") is False: + dropped["event_self_invalid"] += 1 + continue + anchor = event_anchor(content) + if anchor is None: + dropped["event_missing_anchor"] += 1 + continue + ungrounded = {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers} + if ungrounded: + dropped["event_number_not_grounded"] += 1 + continue + entities: list[dict] = [] + for entity in entities_raw: + if not isinstance(entity, dict): + dropped["entity_not_object"] += 1 + continue + if set(entity.keys()) - _ALLOWED_ENTITY_KEYS: + dropped["entity_extra_fields"] += 1 + continue + etype = str(entity.get("type") or "").strip().upper() + ename = str(entity.get("name") or "").strip() + edesc = str(entity.get("description") or "").strip() + if not etype or not ename or not edesc: + dropped["entity_blank"] += 1 + continue + if etype not in _ALLOWED_TYPES: + dropped["entity_type_unknown"] += 1 + continue + entities.append({"type": etype, "name": ename, "description": edesc}) + valid_events.append( + {"title": title, "content": content, "anchor": anchor, "entities": entities} + ) + return valid_events, dict(dropped) + diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py new file mode 100644 index 00000000..f247cde0 --- /dev/null +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -0,0 +1,135 @@ +"""Event-entity KG builder (PMS fork, SAG-adapted). + +来源与许可：核心设计改编自 Zleap-AI/SAG `zleap/sag/modules/extract/{schema,grounding}.py` +与 `prompts/extract_document.yaml` v3.1（MIT，Copyright (c) 2026 sag contributors， +arXiv:2606.15971）。改编点见 PMS fork docs/PMS_PATCHES.md #7 与 Phase 2 方案文档 §3.2。 + +与 LightRAGKGBuilder 的差异： +- 每 chunk 抽取"语义完整的事件"而非三元组：事件 = EVENT 节点 + "事件→参与者" + 星型边（超边的星型编码，n 元关系不拆三元组）； +- 严格响应合同（禁合同外字段、is_valid 自报、无锚 fail-closed 丢弃）； +- 数字接地（SAG grounding 移植）：事件中一切数字必须在原文逐字出现，否则弃； +- 事件节点 id 确定性（内容 SHA），chunk 重叠去重由官方 merge_nodes SEP 合并免费获得。 + +extract() 返回与官方 ({entity_name: [node]}, {(src, tgt): [edge]}) 相同的形状， +merge_nodes / merge_edges / 描述摘要全部复用官方实现。 +""" + +from __future__ import annotations + +import json + +from collections import defaultdict +from typing import Any, Dict, List, Tuple + +from graphgen.bases import BaseGraphStorage, BaseLLMWrapper, Chunk +from graphgen.templates import EVENT_ENTITY_EXTRACTION_PROMPT +from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES +from graphgen.utils import detect_main_language, logger + +from graphgen.models.kg_builder.event_contract import ( + PROJECT_ID_RE, + _ALLOWED_ENTITY_KEYS, + _ALLOWED_EVENT_KEYS, + _ALLOWED_TYPES, + event_anchor, + event_node_name, + numbers_in, + parse_extraction_response, +) + + +class EventEntityKGBuilder: + """事件-实体抽取器：extract → 星型编码节点/边；合并复用官方 LightRAG merge。""" + + def __init__(self, llm_client: BaseLLMWrapper): + from graphgen.models import LightRAGKGBuilder + + self.llm_client = llm_client + self._merger = LightRAGKGBuilder(llm_client=llm_client) + + async def extract( + self, chunk: Chunk + ) -> Tuple[Dict[str, List[dict]], Dict[Tuple[str, str], List[dict]]]: + """单 chunk 事件抽取，返回与官方同形的 (nodes_data, edges_data)。""" + content = chunk.content + language = detect_main_language(content) + entity_types_json = json.dumps(PMS_ENTITY_TYPES, ensure_ascii=False) + request_json = json.dumps( + { + "type": "request", + "data": { + "items": [{"id": 1, "content": content}], + "meta": {"entity_types": "__ENTITY_TYPES__"}, + }, + }, + ensure_ascii=False, + ).replace("__ENTITY_TYPES__", entity_types_json) + # 模板含 JSON 字面大括号，禁用 str.format；用占位符替换注入输入。 + prompt = EVENT_ENTITY_EXTRACTION_PROMPT[language].replace("{input_text}", request_json) + + response = await self.llm_client.generate_answer(prompt) + events, dropped = parse_extraction_response(response, chunk_content=content) + if sum(dropped.values()): + logger.warning( + "Event extraction dropped: %s (chunk=%s)", dropped, chunk.id + ) + if not events: + return {}, {} + + nodes: Dict[str, List[dict]] = defaultdict(list) + edges: Dict[Tuple[str, str], List[dict]] = defaultdict(list) + for event in events: + anchor = event["anchor"] + event_name = event_node_name(anchor, event["title"], event["content"]) + description = f"{event['title']}：{event['content']}" + # 事件节点（EVENT 实体，扁平属性；参与者由星型边表达） + nodes[event_name].append( + { + "entity_type": "EVENT", + "entity_name": event_name, + "description": description, + "source_id": chunk.id, + } + ) + # 参与者实体（去重保序；锚点项目实体若缺失则补齐） + seen: set[str] = set() + participants: list[dict] = [] + for entity in event["entities"]: + if entity["name"] not in seen: + seen.add(entity["name"]) + participants.append(entity) + if anchor not in seen: + participants.append( + { + "type": "PROJECT", + "name": anchor, + "description": f"项目 {anchor}，本事项的锚定项目", + } + ) + seen.add(anchor) + for entity in participants: + nodes[entity["name"]].append( + { + "entity_type": entity["type"], + "entity_name": entity["name"], + "description": entity["description"], + "source_id": chunk.id, + } + ) + edges[(event_name, entity["name"])].append( + { + "src_id": event_name, + "tgt_id": entity["name"], + "description": f"事件参与者：{entity['name']}（{entity['type']}）—— {entity['description']}", + "source_id": chunk.id, + "role": entity["type"], + } + ) + return dict(nodes), dict(edges) + + async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: + return await self._merger.merge_nodes(node_data, kg_instance=kg_instance) + + async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: + return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/__init__.py b/graphgen/models/partitioner/__init__.py index 4306f247..8d5b20bb 100644 --- a/graphgen/models/partitioner/__init__.py +++ b/graphgen/models/partitioner/__init__.py @@ -2,6 +2,7 @@ from .bfs_partitioner import BFSPartitioner from .dfs_partitioner import DFSPartitioner from .ece_partitioner import ECEPartitioner +from .event_join_partitioner import EventJoinPartitioner from .leiden_partitioner import LeidenPartitioner from .quintuple_partitioner import QuintuplePartitioner from .triple_partitioner import TriplePartitioner diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py new file mode 100644 index 00000000..48d51b7b --- /dev/null +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -0,0 +1,113 @@ +"""Event-join partitioner (PMS fork, SAG join semantics). + +来源：共享实体 join 语义翻译自 Zleap-AI/SAG `modules/search/base.py` 的 +entity→event→relation 两跳查询（MIT，arXiv:2606.15971）；离线物化见 Phase 2 方案文档。 + +事件 = EVENT 节点（id 形如 ``event:{anchor}:{hash12}``）+ 星型成员边。分区算法： +1. 种子事件按节点 id 排序（确定性）； +2. 通过共享参与者实体 join 一跳邻居事件； +3. 硬约束：只接受同 project_anchor 事件（簇项目纯度 100%）； +4. 容量上限：max_events_per_community 或事件描述 token 总和。 + +产出官方 ``Community``（nodes = 事件+参与者实体，edges = 星型成员边）， +metadata 携带 event_ids 供出题端 support 校验。 +""" + +import math +from collections import deque +from typing import Any, Iterable, List, Optional, Set, Tuple + +from graphgen.bases import BaseGraphStorage +from graphgen.bases.datatypes import Community + +EVENT_NODE_PREFIX = "event:" + + +class EventJoinPartitioner: + """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" + + def partition( + self, + g: BaseGraphStorage, + max_events_per_community: int = 6, + max_tokens_per_community: int = 4096, + min_events_per_community: int = 1, + **kwargs: Any, + ) -> Iterable[Community]: + events: List[Tuple[str, dict]] = [] + node_dict: dict[str, dict] = {} + for nid, data in g.get_all_nodes(): + node_dict[nid] = data + if str(data.get("entity_type", "")).upper() == "EVENT": + events.append((nid, data)) + events.sort(key=lambda item: item[0]) + + # 参与者 → 事件 倒排索引（共享实体 join key） + events_by_entity: dict[str, List[str]] = {} + star_edges: dict[str, List[str]] = {} + for event_id, _ in events: + star_edges[event_id] = [] + for neighbor in g.get_neighbors(event_id): + star_edges[event_id].append(neighbor) + events_by_entity.setdefault(neighbor, []).append(event_id) + + used: Set[str] = set() + + def _anchor(event_id: str) -> str: + # event:{anchor}:{hash} + return event_id[len(EVENT_NODE_PREFIX):].rsplit(":", 1)[0] + + def _grow(seed: Tuple[str, dict]) -> Optional[Community]: + member_events: List[str] = [] + event_set: Set[str] = set() + member_entities: dict[str, dict] = {} + member_edges: Set[frozenset] = set() + token_sum = 0 + anchor = _anchor(seed[0]) + queue = deque([seed[0]]) + + while queue: + event_id = queue.popleft() + if event_id in event_set: + continue + event_data = node_dict[event_id] + if token_sum + int(event_data.get("length", 0)) > max_tokens_per_community: + continue + event_set.add(event_id) + member_events.append(event_id) + token_sum += int(event_data.get("length", 0)) + for entity_id in star_edges.get(event_id, []): + if entity_id not in member_entities: + entity_data = node_dict.get(entity_id) + if entity_data is not None: + member_entities[entity_id] = entity_data + edge_key = frozenset((event_id, entity_id)) + member_edges.add(edge_key) + # 共享实体 join 一跳扩展；跨锚事件被硬约束拒绝 + for other in events_by_entity.get(entity_id, []): + if other in used or other in event_set: + continue + if _anchor(other) != anchor: + continue + if token_sum + int(node_dict[other].get("length", 0)) > max_tokens_per_community: + continue + queue.append(other) + if len(event_set) >= max_events_per_community: + break + + if len(member_events) < min_events_per_community: + return None + used.update(event_set) + return Community( + id=seed[0], + nodes=member_events + sorted(member_entities), + edges=[tuple(sorted(edge)) for edge in sorted(member_edges, key=sorted)], + metadata={"event_ids": member_events, "anchor": anchor}, + ) + + for event_id, data in events: + if event_id in used: + continue + community = _grow((event_id, data)) + if community: + yield community diff --git a/graphgen/operators/build_kg/build_event_kg.py b/graphgen/operators/build_kg/build_event_kg.py new file mode 100644 index 00000000..a25ca139 --- /dev/null +++ b/graphgen/operators/build_kg/build_event_kg.py @@ -0,0 +1,51 @@ +"""Event-entity KG construction over text chunks (PMS fork).""" + +from __future__ import annotations + +from collections import defaultdict +from typing import List + +from graphgen.bases import BaseLLMWrapper +from graphgen.bases.base_storage import BaseGraphStorage +from graphgen.bases.datatypes import Chunk +from graphgen.models.kg_builder.event_entity_kg_builder import EventEntityKGBuilder +from graphgen.utils import run_concurrent + + +def build_event_entity_kg( + llm_client: BaseLLMWrapper, + kg_instance: BaseGraphStorage, + chunks: List[Chunk], +) -> tuple: + """chunks → 事件星型图（合并/摘要/SEP 血缘全部复用官方 LightRAG merge）。""" + kg_builder = EventEntityKGBuilder(llm_client=llm_client) + + results = run_concurrent( + kg_builder.extract, + chunks, + desc="[2/4]Extracting events and entities from chunks", + unit="chunk", + ) + results = [res for res in results if res] + + nodes = defaultdict(list) + edges = defaultdict(list) + for n, e in results: + for k, v in n.items(): + nodes[k].extend(v) + for k, v in e.items(): + edges[tuple(sorted(k))].extend(v) + + nodes = run_concurrent( + lambda kv: kg_builder.merge_nodes(kv, kg_instance=kg_instance), + list(nodes.items()), + desc="Inserting events and entities into storage", + ) + + edges = run_concurrent( + lambda kv: kg_builder.merge_edges(kv, kg_instance=kg_instance), + list(edges.items()), + desc="Inserting event membership into storage", + ) + + return nodes, edges diff --git a/graphgen/operators/build_kg/build_kg_service.py b/graphgen/operators/build_kg/build_kg_service.py index 19fa9e7a..9a3c484c 100644 --- a/graphgen/operators/build_kg/build_kg_service.py +++ b/graphgen/operators/build_kg/build_kg_service.py @@ -6,6 +6,7 @@ from graphgen.common.init_storage import init_storage from graphgen.utils import logger +from .build_event_kg import build_event_entity_kg from .build_mm_kg import build_mm_kg from .build_text_kg import build_text_kg @@ -27,6 +28,9 @@ def __init__( ) self.build_kwargs = build_kwargs self.max_loop: int = int(self.build_kwargs.get("max_loop", 3)) + # PMS fork（Phase 2）：文本 KG 抽取方法分发。默认 light_rag 与官方逐字节 + # 一致；event_entity 走事件-实体星型编码（见 models/kg_builder/event_entity_kg_builder.py）。 + self.kg_method: str = str(self.build_kwargs.get("kg_method", "light_rag")) def process(self, batch: list) -> Tuple[list, dict]: """ @@ -49,6 +53,15 @@ def process(self, batch: list) -> Tuple[list, dict]: if len(text_chunks) == 0: logger.info("All text chunks are already in the storage") + elif self.kg_method == "event_entity": + logger.info("[Text Event and Entity Extraction] processing ...") + text_nodes, text_edges = build_event_entity_kg( + llm_client=self.llm_client, + kg_instance=self.graph_storage, + chunks=text_chunks, + ) + nodes += text_nodes + edges += text_edges else: logger.info("[Text Entity and Relation Extraction] processing ...") text_nodes, text_edges = build_text_kg( diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 52359b29..2179a1eb 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -130,18 +130,25 @@ def process(self, batch: list) -> Tuple[list, dict]: unit="batch", ) + from graphgen.models.generator.support import validate_support + meta_updates = {} final_results = [] - for input_trace_id, qa_pairs in zip( - [item["_trace_id"] for item in batch], results - ): + for item, qa_pairs in zip(batch, results): if not qa_pairs: continue + # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 + # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 + node_names = {node[0] for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and node} for qa_pair in qa_pairs: + support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None + support = validate_support(support, node_names) + if support: + qa_pair["support"] = support res = self.generator.format_generation_results( qa_pair, output_data_format=self.data_format ) res["_trace_id"] = self.get_trace_id(res) final_results.append(res) - meta_updates.setdefault(input_trace_id, []).append(res["_trace_id"]) + meta_updates.setdefault(item["_trace_id"], []).append(res["_trace_id"]) return final_results, meta_updates diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index b55c5159..6e7ddff3 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -38,6 +38,10 @@ def __init__( from graphgen.models import DFSPartitioner self.partitioner = DFSPartitioner() + elif method == "event_join": + from graphgen.models import EventJoinPartitioner + + self.partitioner = EventJoinPartitioner() elif method == "ece": # before ECE partitioning, we need to: # 'quiz' and 'judge' to get the comprehension loss if unit_sampling is not random @@ -87,6 +91,10 @@ def generator(): "nodes": b[0], "edges": b[1], } + # PMS fork（Phase 2）：事件簇附带 event_ids，供出题端 support 校验。 + event_ids = list(getattr(community, "metadata", {}) or {}).get("event_ids") or [] + if event_ids: + result["events"] = event_ids result["_trace_id"] = self.get_trace_id(result) yield result logger.info("Total communities partitioned: %d", count) diff --git a/graphgen/templates/__init__.py b/graphgen/templates/__init__.py index 0b6a865d..dd2cb60d 100644 --- a/graphgen/templates/__init__.py +++ b/graphgen/templates/__init__.py @@ -13,7 +13,12 @@ TF_GENERATION_PROMPT, VQA_GENERATION_PROMPT, ) -from .kg import KG_EXTRACTION_PROMPT, KG_SUMMARIZATION_PROMPT, MMKG_EXTRACTION_PROMPT +from .kg import ( + EVENT_ENTITY_EXTRACTION_PROMPT, + KG_EXTRACTION_PROMPT, + KG_SUMMARIZATION_PROMPT, + MMKG_EXTRACTION_PROMPT, +) from .rephrasing import STYLE_CONTROLLED_REPHRASING_PROMPTS from .search_judgement import SEARCH_JUDGEMENT_PROMPT from .statement_judgement import STATEMENT_JUDGEMENT_PROMPT diff --git a/graphgen/templates/kg/__init__.py b/graphgen/templates/kg/__init__.py index ea865ce6..44c48f5e 100644 --- a/graphgen/templates/kg/__init__.py +++ b/graphgen/templates/kg/__init__.py @@ -1,3 +1,4 @@ +from .event_entity_extraction import EVENT_ENTITY_EXTRACTION_PROMPT from .kg_extraction import KG_EXTRACTION_PROMPT from .kg_summarization import KG_SUMMARIZATION_PROMPT from .mm_kg_extraction import MMKG_EXTRACTION_PROMPT diff --git a/graphgen/templates/kg/event_entity_extraction.py b/graphgen/templates/kg/event_entity_extraction.py new file mode 100644 index 00000000..e74c1be8 --- /dev/null +++ b/graphgen/templates/kg/event_entity_extraction.py @@ -0,0 +1,113 @@ +# pylint: disable=C0301 +"""事件-实体抽取提示词（SAG extract_document v3.1 改编，MIT）。 + +来源：Zleap-AI/SAG `zleap/sag/prompts/extract_document.yaml`（arXiv 2606.15971）。 +改编点（PMS project-query 场景）： +- 事件必须含完整项目编号锚点（P-/I- + ≥6 位数字），无锚事件直接不产出； +- 字段锁定 / 反查禁止 / 越界语义禁止（与 pms_policy 出题铁律同源）; +- 数字接地：事件中一切数字必须在原文出现； +- 只输出扁平事项（不用 children 层级），便于星型编码进图。 +响应是严格 JSON 合同：禁合同外字段，无效字段/无锚/数字不接地一律丢弃并计数。 +""" + +TEMPLATE_ZH: str = """## Role + +你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 +构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 + +## 事项提取原则 + +- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 +- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， + 保留必要的数字和结论。 +- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 +- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 +- 不得把外部常识、文件名或模型推测写成原文事实。 +- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 + （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 + 编号的事项不得输出。 +- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 +- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 + 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 +- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， + 不得写成"某人属于哪个部门"等组织反查事实。 + +## 实体提取原则 + +- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， + 没有合适类型时不要自造类型。 +- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 +- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 +- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 +- 同名实体保持一致；不同实体不得因名称相近而合并。 + +## 事实边界（fail-closed） + +- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 +- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； + 找不到就删掉该表述或放弃该事项。 +- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 + +## Task + +按以下顺序执行，不能跳步： + +1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； + 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 +2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > + 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 +3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 + 编号锚点、禁止推理说明。 +4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 +5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； + 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 + 全部片段为噪音或信息不足时返回 items: []。 + +## Input + +输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], +"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} + +## Output requirements + +- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： + {"type": "response", "data": {"items": [ + {"title": "...", "content": "...", + "entities": [{"type": "...", "name": "...", "description": "..."}], + "is_valid": true} + ]}} +- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null + 或制造占位事项。 +- 相对时间（如"下月"）保留原文，不自行换算。 + +给定输入： +{input_text} + +输出： +""" + +#: 紧凑 JSON 请求包装（与 SAG request 合同同形；PMS 单 chunk 单 items） +INPUT_FORMAT: dict = { + "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' + '"meta": {{"entity_types": {entity_types}}}}}}}', +} + +#: PMS 实体类型集（SAG 5W1H 类型集收敛到 project-query 可查询维度） +PMS_ENTITY_TYPES: list[dict] = [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色，如二级项目经理、销售工程师"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态，如售前、执行"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"}, +] + +EVENT_ENTITY_EXTRACTION_PROMPT = { + "en": TEMPLATE_ZH, + "zh": TEMPLATE_ZH, + "FORMAT": { + "entity_types": None, # 运行时注入 PMS_ENTITY_TYPES + }, +} diff --git a/graphgen/templates/profiles/pms_policy.v3/atomic.json b/graphgen/templates/profiles/pms_policy.v3/atomic.json new file mode 100644 index 00000000..890a911a --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v3/atomic.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n" +} diff --git a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json new file mode 100644 index 00000000..6f8271ba --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" +} From 194f06b6b1a307ff12c6d1402b7a5d2c78adf9c0 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 9 Oct 2026 10:12:11 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(pms):=20Phase2=20=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=9B=9B=E7=BC=BA=E9=99=B7=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=20+=20=E4=BA=8B=E4=BB=B6/=E5=8F=82=E4=B8=8E=E8=80=85=E7=A1=AE?= =?UTF-8?q?=E5=AE=9A=E6=80=A7=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - event_contract 抽为纯 stdlib 模块（rft 契约测试可按路径加载） - builder 补回 json 导入；EVENT 节点/成员边/参与者实体确定性 SEP 合并，不再触发 LightRAG LLM 摘要（摘要器曾把 event id 扩写成伪业务事实污染出题上下文） - EventJoinPartitioner 继承 BasePartitioner（community2batch）；models lazy export 补齐 - v3 profile support 大括号转义（str.format KeyError）；分区服务移除 metadata 依赖 （support 校验以持久化分区节点集合为准，JSONL 往返一致） - 真实 10 记录端到端 smoke 全通：7 分区→7 题全锚点合规、cited 全部通过分区校验 Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 8 +- .../kg_builder/event_entity_kg_builder.py | 73 ++++++++++++++++++- .../partitioner/event_join_partitioner.py | 5 +- .../operators/generate/generate_service.py | 6 +- .../operators/partition/partition_service.py | 4 - .../profiles/pms_policy.v3/atomic.json | 4 +- .../profiles/pms_policy.v3/multi_hop.json | 4 +- 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 88d2fb8e..f69d7631 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -65,12 +65,16 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc - `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 - `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， - 边属性含角色）；合并/摘要/SEP 血缘复用官方 LightRAG merge。 + 边属性含角色）。实体描述/普通边继续复用官方合并；**EVENT 节点与事件成员边用确定性 + SEP 合并、不走 LLM 摘要**（SAG 事件是事实文本，不能让 merge summarizer 把 event_id + 扩写成“项目身份/分工总结”污染出题上下文）。事件描述限长 4,000 字符，且原文链以 + `source_id` SEP 保持闭合。 - `operators/build_kg/`：`kg_method: event_entity` 分发（默认 light_rag 与官方逐字节一致）。 - `models/partitioner/event_join_partitioner.py`：共享实体 join + 同锚硬约束的事件簇分区 （SAG join 语义离线物化），产出官方 `Community`，metadata 携带 event_ids。 - `models/generator/support.py` + atomic/multi_hop 解析 + GenerateService 校验： - `{"cited": [...]}` 引用必须存在于本分区节点集合，失败剥离保留题面。 + `{"cited": [...]}` 引用必须存在于本分区节点集合（EVENT 节点名即事件审计 + ID，不依赖自定义 Community metadata 以保证 JSONL cache 往返一致），失败剥离保留题面。 - `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 ## 验收 diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py index f247cde0..1aa44489 100644 --- a/graphgen/models/kg_builder/event_entity_kg_builder.py +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -27,6 +27,9 @@ from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES from graphgen.utils import detect_main_language, logger +_EVENT_DESC_CAP = 4000 +_ENTITY_DESC_CAP = 2000 + from graphgen.models.kg_builder.event_contract import ( PROJECT_ID_RE, _ALLOWED_ENTITY_KEYS, @@ -129,7 +132,75 @@ async def extract( return dict(nodes), dict(edges) async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: - return await self._merger.merge_nodes(node_data, kg_instance=kg_instance) + entity_name, node_list = node_data + if node_list and str(node_list[0].get("entity_type", "")).upper() == "EVENT": + return await self._merge_event_node(entity_name, node_list, kg_instance) + # 星型事件参与者是图投影，不需要再次 LLM 摘要；只做确定性去重合并， + # 避免 LightRAG summarizer 把跨事件上下文扩写成伪业务事实。 + return await self._merge_entity_node(entity_name, node_list, kg_instance) + + async def _merge_entity_node( + self, entity_name: str, node_list: list, kg_instance: BaseGraphStorage + ) -> dict: + existing = kg_instance.get_node(entity_name) or {} + all_rows = list(node_list) + if existing: + all_rows.append(existing) + types = [str(dp.get("entity_type") or "KEYWORD") for dp in all_rows] + entity_type = sorted(set(types), key=lambda t: (-types.count(t), t))[0] + descriptions = sorted({str(dp.get("description") or "") for dp in all_rows if dp.get("description")}) + merged = "".join(descriptions)[:_ENTITY_DESC_CAP] + source_ids = sorted({sid for dp in all_rows for sid in str(dp.get("source_id", "")).split("") if sid}) + payload = { + "entity_type": entity_type, + "entity_name": entity_name, + "description": merged, + "source_id": "".join(source_ids), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_node(entity_name, node_data=payload) + return payload + + async def _merge_event_node( + self, entity_name: str, node_list: list, kg_instance: BaseGraphStorage + ) -> dict: + """事件节点确定性合并：SEP 去重拼接（截断），不走 LLM 摘要—— + 官方摘要器会把事件 id 当实体名写进描述，污染出题上下文。""" + merged = "".join(sorted({dp["description"] for dp in node_list})) + if len(merged) > _EVENT_DESC_CAP: + merged = merged[:_EVENT_DESC_CAP] + node_data_dict = { + "entity_type": "EVENT", + "entity_name": entity_name, + "description": merged, + "source_id": "".join( + sorted({sid for dp in node_list for sid in str(dp.get("source_id", "")).split("") if sid} + | {sid for sid in str((kg_instance.get_node(entity_name) or {}).get("source_id", "")).split("") if sid}) + ), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_node(entity_name, node_data=node_data_dict) + return node_data_dict async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: + (src_id, tgt_id), edge_list = edges_data + if src_id.startswith("event:") or tgt_id.startswith("event:"): + # 事件成员边同样确定性合并，不做 LLM 摘要 + merged = "".join(sorted({dp["description"] for dp in edge_list})) + if len(merged) > _EVENT_DESC_CAP: + merged = merged[:_EVENT_DESC_CAP] + source_ids = { + sid for dp in edge_list for sid in str(dp.get("source_id", "")).split("") if sid + } | { + sid for sid in str((kg_instance.get_edge(src_id, tgt_id) or {}).get("source_id", "")).split("") if sid + } + edge_data = { + "src_id": src_id, + "tgt_id": tgt_id, + "description": merged, + "source_id": "".join(sorted(source_ids)), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_edge(src_id, tgt_id, edge_data=edge_data) + return edge_data return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py index 48d51b7b..3414ff16 100644 --- a/graphgen/models/partitioner/event_join_partitioner.py +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -17,13 +17,13 @@ from collections import deque from typing import Any, Iterable, List, Optional, Set, Tuple -from graphgen.bases import BaseGraphStorage +from graphgen.bases import BaseGraphStorage, BasePartitioner from graphgen.bases.datatypes import Community EVENT_NODE_PREFIX = "event:" -class EventJoinPartitioner: +class EventJoinPartitioner(BasePartitioner): """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" def partition( @@ -102,7 +102,6 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: id=seed[0], nodes=member_events + sorted(member_entities), edges=[tuple(sorted(edge)) for edge in sorted(member_edges, key=sorted)], - metadata={"event_ids": member_events, "anchor": anchor}, ) for event_id, data in events: diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 2179a1eb..e6676fe6 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -139,7 +139,11 @@ def process(self, batch: list) -> Tuple[list, dict]: continue # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 - node_names = {node[0] for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and node} + node_names = { + node[0] + for node in item.get("nodes", []) + if isinstance(node, (list, tuple)) and node + } for qa_pair in qa_pairs: support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None support = validate_support(support, node_names) diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index 6e7ddff3..db355769 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -91,10 +91,6 @@ def generator(): "nodes": b[0], "edges": b[1], } - # PMS fork（Phase 2）：事件簇附带 event_ids，供出题端 support 校验。 - event_ids = list(getattr(community, "metadata", {}) or {}).get("event_ids") or [] - if event_ids: - result["events"] = event_ids result["_trace_id"] = self.get_trace_id(result) yield result logger.info("Total communities partitioned: %d", count) diff --git a/graphgen/templates/profiles/pms_policy.v3/atomic.json b/graphgen/templates/profiles/pms_policy.v3/atomic.json index 890a911a..1f1bb4ad 100644 --- a/graphgen/templates/profiles/pms_policy.v3/atomic.json +++ b/graphgen/templates/profiles/pms_policy.v3/atomic.json @@ -1,4 +1,4 @@ { - "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n", - "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n" + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定图谱片段：\n{context}\n\n输出：\n" } diff --git a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json index 6f8271ba..6bd06a1a 100644 --- a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json +++ b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json @@ -1,4 +1,4 @@ { - "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", - "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" } From c5ef453aa661776b757902a990c529fdab2bfc9a Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Sat, 10 Oct 2026 10:11:04 +0000 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E9=80=9A=E7=94=A8=E5=8C=96GraphGen?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E6=8A=BD=E5=8F=96=E5=B9=B6=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E5=A4=9A=E9=A1=B9=E7=9B=AE=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 引入版本化事件抽取profile，将实体类型、prompt、锚点和数字接地策略从PMS专用代码解耦。event_join基于显式锚点属性隔离，生成prompt隐藏内部事件ID并把support引用映射回canonical节点。\n\n新增多项目SDK与不可变tag发布文档，修正安装包资源声明和过时的partition integration测试；保留LightRAG+ECE默认路径及PMS事件ID兼容。完成真实非PMS抽取、分区和问答生成smoke，默认生成结果未泄漏内部事件ID。\n\nCo-Authored-By: Claude Opus 5.5 --- MANIFEST.in | 2 + README.md | 6 + README_zh.md | 6 + docs/MULTI_PROJECT_SDK.md | 84 ++++++++++++ docs/PMS_PATCHES.md | 15 +- docs/RELEASE_NOTES-1.0.0.md | 23 ++++ docs/RELEASING.md | 43 ++++++ graphgen/_version.py | 2 +- graphgen/models/generator/support.py | 87 ++++++++++-- graphgen/models/kg_builder/event_contract.py | 93 ++++++++----- .../kg_builder/event_entity_kg_builder.py | 105 +++++++------- .../partitioner/event_join_partitioner.py | 38 +++--- graphgen/operators/build_kg/build_event_kg.py | 17 ++- .../operators/build_kg/build_kg_service.py | 9 +- .../operators/generate/generate_service.py | 20 +-- .../operators/partition/partition_service.py | 5 + graphgen/templates/event_profile_loader.py | 129 ++++++++++++++++++ .../example_event.v1/profile.json | 13 ++ .../example_event.v1/prompt.zh.txt | 28 ++++ .../generic_event.v1/profile.json | 12 ++ .../generic_event.v1/prompt.zh.txt | 27 ++++ .../event_profiles/pms_event.v1/profile.json | 18 +++ .../event_profiles/pms_event.v1/prompt.zh.txt | 74 ++++++++++ .../templates/kg/event_entity_extraction.py | 118 ++-------------- requirements-dev.txt | 2 +- setup.py | 14 +- .../partitioner/test_bfs_partitioner.py | 32 ++--- .../partitioner/test_dfs_partitioner.py | 34 ++--- .../partitioner/test_ece_partitioner.py | 65 ++++----- tests/test_event_profiles.py | 20 +++ tests/test_package_profiles.py | 30 ++++ 31 files changed, 852 insertions(+), 319 deletions(-) create mode 100644 docs/MULTI_PROJECT_SDK.md create mode 100644 docs/RELEASE_NOTES-1.0.0.md create mode 100644 docs/RELEASING.md create mode 100644 graphgen/templates/event_profile_loader.py create mode 100644 graphgen/templates/event_profiles/example_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt create mode 100644 graphgen/templates/event_profiles/generic_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt create mode 100644 graphgen/templates/event_profiles/pms_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt create mode 100644 tests/test_event_profiles.py create mode 100644 tests/test_package_profiles.py diff --git a/MANIFEST.in b/MANIFEST.in index d1a9003b..c8572113 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include requirements.txt include graphgen/configs/*.yaml recursive-include graphgen/configs *.yaml +recursive-include graphgen/templates/profiles *.json +recursive-include graphgen/templates/event_profiles *.json *.txt diff --git a/README.md b/README.md index 2fdff54b..86e5d4fe 100644 --- a/README.md +++ b/README.md @@ -374,3 +374,9 @@ This project is licensed under the [Apache License 2.0](LICENSE). ## 📅 Star History [![Star History Chart](https://star-history.dera.page/svg?repos=InternScience/GraphGen&type=Date)](https://star-history.dera.page/#InternScience/GraphGen&Date) + +## Maintained fork: multi-project SDK and event profiles + +This repository also maintains the `Leon-Algo/GraphGen` fork. Fork-specific reusable capabilities, project-owned event extraction profiles, installation guidance, and compatibility boundaries are documented in [docs/MULTI_PROJECT_SDK.md](docs/MULTI_PROJECT_SDK.md). The immutable version/tag and release verification policy is in [docs/RELEASING.md](docs/RELEASING.md); the fork patch ledger and SAG attribution are in [docs/PMS_PATCHES.md](docs/PMS_PATCHES.md). + +Fork profiles are opt-in. The default LightRAG + ECE workflow remains unchanged, and PMS-specific prompts are not a default policy for other projects. diff --git a/README_zh.md b/README_zh.md index fc1bc8e0..6938b754 100644 --- a/README_zh.md +++ b/README_zh.md @@ -375,3 +375,9 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期 [![Star History Chart](https://star-history.dera.page/svg?repos=InternScience/GraphGen&type=Date)](https://star-history.dera.page/#InternScience/GraphGen&Date) + +## 自维护 Fork：多项目 SDK 与事件 Profile + +本仓库同时维护 `Leon-Algo/GraphGen` fork。跨项目可复用能力、项目自有事件抽取 profile、安装与兼容边界见[多项目 SDK 指南](docs/MULTI_PROJECT_SDK.md)；不可变版本/tag 与发布验证规则见[发布规范](docs/RELEASING.md)；fork 补丁及 SAG 来源归属见[补丁台账](docs/PMS_PATCHES.md)。 + +Fork 能力均显式 opt-in：默认 LightRAG + ECE 管线保持不变，PMS 专用提示词不会自动成为其他项目的策略。 diff --git a/docs/MULTI_PROJECT_SDK.md b/docs/MULTI_PROJECT_SDK.md new file mode 100644 index 00000000..ee265255 --- /dev/null +++ b/docs/MULTI_PROJECT_SDK.md @@ -0,0 +1,84 @@ +# GraphGen Fork: reusable Graph + SAG event capabilities + +This repository is the independently maintained `Leon-Algo/GraphGen` fork of GraphGen. It remains compatible with the upstream project where documented, while shipping opt-in fork capabilities. The upstream PR is not a release dependency: projects consume reviewed fork commits/tags directly. + +## What is reusable + +- **GraphGen foundation:** pipeline/operators, storage integration, generators, and the default LightRAG + ECE route. +- **Fork capabilities:** versioned generation prompt profiles, output gates, generation concurrency controls, and the opt-in event/entity star-graph builder plus event-join partitioner. +- **SAG-derived event mechanics:** event extraction response contract, numeric grounding, star encoding (one event node linked to participants), deterministic merging, and shared-participant joins. Adapted assets and code retain their upstream license/source attribution in `docs/PMS_PATCHES.md`. + +These mechanisms do not define a universal business ontology. A consuming project selects a versioned event extraction profile and its own generation prompt profile. The `pms_event.v1` and `pms_policy.v2/v3` profiles are PMS-specific examples and are never implicitly applied to unrelated projects. `generic_event.v1` and `example_event.v1` are synthetic examples, not validated domain policies. + +## Install a reviewed fork version + +For a reproducible project, pin an immutable commit in the consuming project's `pyproject.toml` and commit its lock file: + +```toml +[project] +dependencies = [ + "graphg @ git+https://github.com/Leon-Algo/GraphGen.git@", +] +``` + +Resolve and lock with `uv lock` (or add it with `uv add`). Avoid moving branch refs in production. For short-lived development only, install `git+https://github.com/Leon-Algo/GraphGen.git@pms/main`. The distribution name is `graphg`; Python imports use `graphgen`. + +## Select event extraction explicitly + +The default `kg_method` remains `light_rag`; the normal ECE partitioner remains unchanged. To use the event route, configure the build node explicitly: + +```yaml +- id: build_kg + op_name: build_kg + type: map + params: + kg_method: event_entity + event_profile: generic_event.v1 + +- id: partition + op_name: partition + type: map + params: + method: event_join + method_params: + event_entity_type: EVENT + anchor_attribute: anchor + require_same_anchor: false + max_events_per_community: 6 + max_tokens_per_community: 4096 +``` + +For a project-specific profile, copy a profile directory into a project-controlled location and point `event_profile` to the absolute or explicitly configured `profile.json` path; do not rely on the process working directory. Never edit a profile already used by a released run. The loader validates the manifest and prompt, fails closed on missing/invalid inputs, and records SHA-256 fingerprints on the loaded profile object. + +An event profile contains: + +- `schema_version` and immutable `profile_id`; +- `prompt_file` with the required `{input_text}` placeholder; +- `entity_types` allowlist used both in prompt input and response validation; +- `anchor` with required/optional status, optional regex, and optional anchor entity type; +- `grounding.number_grounding`; +- `graph.event_entity_type` and `graph.legacy_event_id`. + +Regex capture group 1 is the anchor when present; otherwise the complete match is used. Event nodes carry the extracted anchor as an explicit string attribute; the partitioner never parses it from the node ID. `require_same_anchor: true` rejects missing anchors and prevents joins across anchor values. Set it to false only when cross-event joining without anchor isolation is acceptable. Limits remain explicit partition parameters. + +The packaged `generic_event.v1` profile demonstrates an anchor-free schema; `example_event.v1` demonstrates a custom anchor and domain entity set. They are fixtures, not a claim that a particular project's extraction policy has been validated. PMS workflows should explicitly select `pms_event.v1` and the corresponding PMS generation policy; legacy configurations with `kg_method: event_entity` and no profile retain the PMS-compatible profile. + +## Generation profiles and gates + +Generation profiles use the independent `prompt_profile` parameter (built-in profile ID or explicit profile directory); do not assume it selects event extraction policy. PMS output gate `pms_project_anchor` is opt-in and is not a generic default. Each project should own its prompt policy and only enable domain gates it has validated. + +## Profile authoring checklist + +1. Define a versioned profile ID and the consuming project's allowed entity types. +2. Write prompt instructions for that domain; include `{input_text}` exactly once or as required by the application. +3. Set an explicit anchor regex or `required: false`; set `anchor_entity_type` only if it is in the entity allowlist. +4. Decide numeric grounding and whether event_join must enforce same-anchor isolation. +5. Test supported, missing, malformed, ungrounded, and unknown-entity responses with a fake LLM before real model smoke. +6. Test extraction → star graph → partition on representative project data, then pin the profile bytes and GraphGen commit in the consuming project. + +## Upgrade and compatibility policy + +- A released tag and profile directory are immutable. Fixes and policy changes get a new commit and a new tag/profile ID; do not retarget tags or edit released profile contents in place. +- Consumers pin a full commit SHA or immutable release tag and commit their lock file. Upgrade by changing the pin deliberately, rerunning tests/smokes, and reviewing GraphGen's release notes and `docs/PMS_PATCHES.md`. +- `light_rag` remains the default. `event_entity` is opt-in. PMS profile defaults exist only for backward compatibility of the PMS fork integration; new projects should always specify their profile explicitly. +- Fork release/tag format, validation gate, attribution, upstream base, and exact release evidence are maintained in `docs/RELEASING.md`. diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index f69d7631..54736cad 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -60,8 +60,7 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_reference/`，方案见 PMS 主仓 `docs/2026-09-18-graphgen-phase2-sag-event-entity-plan.md` §3.2 复用地图）： -- `templates/kg/event_entity_extraction.py`：事件抽取提示词（SAG extract_document v3.1 改编 + - PMS 锚点/字段锁定/数字接地铁律）。 +- `templates/kg/event_entity_extraction.py` 保留旧 Python 常量的兼容导出；实际策略由版本化 event profile 资源提供。 - `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 - `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， @@ -77,10 +76,20 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc ID，不依赖自定义 Community metadata 以保证 JSONL cache 往返一致），失败剥离保留题面。 - `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 +## #8 通用事件抽取 Profile 与多项目 SAG 能力 + +- 将事件抽取策略从 builder/契约代码中抽离为 `templates/event_profiles//` 下版本化 manifest + prompt。内置 `pms_event.v1` 保留原始 PMS prompt、8 类实体、P-/I- 锚点和旧事件 ID；`generic_event.v1` 与 `example_event.v1` 是机制示例/测试 fixture，不代表外部领域已验证策略。 +- `templates/event_profile_loader.py` 校验 schema、实体类型、锚点 regex、prompt 占位符及图设置，暴露 manifest/prompt SHA-256；显式无效 profile fail-closed。profile 从包资源读取，不依赖当前工作目录。 +- `build_kg.params.event_profile` 传入事件 builder。老 PMS 配置缺省解析为 `pms_event.v1`；新项目应显式传入自己的 profile。`light_rag` 默认保持不变。 +- `event_contract.py` 仍保证严格响应 JSON、实体 allowlist 和可选数字接地；实体类型、锚点是否必需及锚点模式由 profile 驱动。无锚模式不自动合成 PROJECT/其他锚点实体。 +- 事件节点写入显式 `anchor` 属性。`event_join` 按 `event_entity_type` 和 `anchor_attribute` 配置工作，`require_same_anchor` 控制同锚硬隔离；不再从 ID 推断锚点。PMS profile 使用旧 ID 以兼容既有图缓存，新 profile 使用不编码 domain anchor 的 ID。 +- `setup.py` 与 `MANIFEST.in` 将生成 profile JSON、事件 manifest/prompt 作为 `graphgen` wheel/sdist 包资源。 +- 多项目安装、profile authoring 与升级见 `docs/MULTI_PROJECT_SDK.md`；fork 不可变 tag 与发布门禁见 `docs/RELEASING.md`。 + ## 验收 - 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； - PMS 主仓库 `tests/test_pms_graphgen_policy_generation.py`、 `tests/test_run_graphgen_defaults.py` 为契约回归（rft 环境按路径加载 纯 stdlib 模块校验）； -- 真实生成由 GraphGen venv smoke 验收。 +- 事件 profile 合同和真实生成由 GraphGen venv smoke 验收。 diff --git a/docs/RELEASE_NOTES-1.0.0.md b/docs/RELEASE_NOTES-1.0.0.md new file mode 100644 index 00000000..4297347c --- /dev/null +++ b/docs/RELEASE_NOTES-1.0.0.md @@ -0,0 +1,23 @@ +# GraphGen Fork 1.0.0 — initial multi-project SDK release + +Tag: `graphgen-fork-v1.0.0` (to be created only after passing release validation). + +Distribution version: `1.0.0` (`graphg`). Upstream base: `InternScience/GraphGen` commit `3a3eb097318d34b07f6f31cb60de5b20712d65ce`. + +## Highlights + +- Profile-driven event/entity extraction: projects supply entity types, prompt, anchor policy, numeric grounding and event graph identity without patching GraphGen source. +- Built-in `pms_event.v1` preserves PMS extraction behavior and legacy event IDs; `generic_event.v1` and `example_event.v1` are synthetic examples only. +- `event_join` reads explicit anchor node attributes and supports opt-in same-anchor isolation independently of event ID encoding. +- Default LightRAG + ECE remains unchanged; event extraction is opt-in. +- Event and generation prompt profile assets are included in wheel and sdist and load through package resources. +- See [multi-project SDK guide](MULTI_PROJECT_SDK.md), [release policy](RELEASING.md), and [patch ledger](PMS_PATCHES.md). + +## Compatibility and validation record + +GraphGen partition integration: 12 passed after correcting tests to the synchronous storage/partitioner APIs. Profile, fake-LLM builder/partition, and package tests: 12 passed. PMS contract regression: `tests/test_pms_graphgen_policy_generation.py` — 23 passed, 1 skipped. +A real one-chunk GraphGen run using the configured PMS launcher/model pool completed extraction → event_join → COT generation. The synthetic non-PMS event anchored on `CASE-A7`; the graph contained the event and two participants; the final QA identified the operator and completed mission, included no internal event hash, and the process exited 0. The run used a synthetic fixture and validates integration mechanics, not any production project's domain accuracy. + +The existing GraphGen partitioner integration suite could not collect because its tests import an unexported `NetworkXStorage` from `graphgen.models`. An unrelated GraphGen extraction e2e test failed at Ray GCS startup (`Could not read 'temp_dir' from GCS`). These environment/baseline failures are not represented as passing tests. + +SAG-derived code/prompt attribution and MIT licensing are tracked in `docs/PMS_PATCHES.md`. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 00000000..44c02f7f --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,43 @@ +# GraphGen fork release and tagging policy + +The fork is an independently versioned Python distribution (`graphg`) maintained at `Leon-Algo/GraphGen`. Upstream merge status does not control fork releases. Release tags are immutable identifiers for exact commits; never move or reuse a published tag. + +## Tag format + +Use `graphgen-fork-vMAJOR.MINOR.PATCH`: + +- **MAJOR**: breaking config/profile/schema or graph identity changes requiring consumer migration. +- **MINOR**: backward-compatible capability or new profile/API. +- **PATCH**: backward-compatible bug fix or documentation/packaging correction. + +PMS domain profiles are independently versioned by their profile IDs and are not implied by a package release. Profiles referenced by production configs must not be edited after release; changes get a new profile ID. + +## Release gate + +Before tagging: + +1. Review the fork diff and working tree; ensure no unrelated user changes are included. +2. Run deterministic profile/contract/builder/partitioner tests, relevant upstream GraphGen tests, and PMS compatibility tests. +3. Run a non-PMS extraction → star graph → event_join smoke and a PMS compatibility smoke; confirm default LightRAG + ECE behavior is unchanged. +4. Build wheel and sdist in a disposable directory; inspect both for prompt/profile assets and install the wheel into a clean environment to load each built-in profile. +5. Run `git diff --check`, record exact commit, test commands/results, profile hashes and upstream base. + +If any required gate fails, do not tag or push the release. + +## Release sequence + +1. Merge only reviewed release files into the intended fork branch; preserve unrelated local changes. +2. Commit with a concise message and verify the final commit SHA and clean release diff. +3. Confirm the desired tag is unused locally and on the fork remote. +4. Create the tag on that exact commit, then push the intended branch and tag to the `fork` remote (`https://github.com/Leon-Algo/GraphGen.git`). Never push the fork release tag to upstream by accident. +5. Publish release notes that include: distribution version, tag and commit SHA, upstream base commit, key behavior/config changes, profile IDs and asset hashes, compatibility/migration notes, license/source attribution, test and smoke evidence, and any known limitations. + +Example commands (replace placeholders only after verification): + +```bash +git tag -a graphgen-fork-v1.0.0 -m "GraphGen fork 1.0.0" +git push fork pms/main +git push fork graphgen-fork-v1.0.0 +``` + +Consumers should pin the release tag or preferably full commit SHA and lock their environment. Tag creation/push is an externally visible release action and must occur only after the release gate passes and the maintainer authorizes publication. diff --git a/graphgen/_version.py b/graphgen/_version.py index a7ae27a2..43fe2c19 100644 --- a/graphgen/_version.py +++ b/graphgen/_version.py @@ -1,6 +1,6 @@ from typing import Tuple -__version__ = "0.1.0.post20250930" +__version__ = "1.0.0" short_version = __version__ diff --git a/graphgen/models/generator/support.py b/graphgen/models/generator/support.py index 810f65d2..d66dd633 100644 --- a/graphgen/models/generator/support.py +++ b/graphgen/models/generator/support.py @@ -1,10 +1,4 @@ -""" 审计块解析（PMS fork，Phase 2 升级 C）。 - -生成器可在 / 之后输出 -``{"cited": ["节点或事件名称", ...]}``； -本模块只负责解析，有效性校验（cited 是否真存在于出题分区）在 -GenerateService.process 中按分区的节点集合执行，失败剥离 support 保留题面。 -""" +"""Parse support blocks and build a display-safe graph view for generators.""" from __future__ import annotations @@ -16,7 +10,7 @@ def parse_support(response: str) -> Optional[dict[str, Any]]: - """从模型响应解析 support 块；缺失/畸形返回 None（不影响题面解析）。""" + """Parse support XML; malformed or absent blocks are ignored.""" match = _SUPPORT_RE.search(response or "") if not match: return None @@ -24,16 +18,85 @@ def parse_support(response: str) -> Optional[dict[str, Any]]: payload = json.loads(match.group(1).strip()) except json.JSONDecodeError: return None - if not isinstance(payload, dict) or set(payload.keys()) != {"cited"}: + if not isinstance(payload, dict) or set(payload) != {"cited"}: return None cited = payload.get("cited") - if not isinstance(cited, list) or not all(isinstance(c, str) and c.strip() for c in cited): + if not isinstance(cited, list) or not all(isinstance(value, str) and value.strip() for value in cited): return None - return {"cited": [c.strip() for c in cited]} + return {"cited": [value.strip() for value in cited]} + + +def _is_event_node(node: tuple[str, dict]) -> bool: + return str(node[1].get("graph_role", "")).lower() == "event" + + +def _event_display_name(node: tuple[str, dict]) -> str: + title = str(node[1].get("description", "")).split("：", 1)[0].strip() + return title or "事件" + + +def build_generation_view(item: dict[str, Any]) -> dict[str, Any]: + """Replace internal event IDs with readable names for generation only. + + Canonical node/edge records remain unchanged for lineage. Duplicate event + titles receive deterministic suffixes; support names are mapped back and + validated against the original partition node IDs. + """ + original_nodes = [node for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and len(node) == 2] + event_nodes = [node for node in original_nodes if _is_event_node(node)] + used_names = { + str(node[0]) for node in original_nodes if not _is_event_node(node) + } + display_by_id: dict[str, str] = {} + for node in sorted(event_nodes, key=lambda row: str(row[0])): + base = _event_display_name(node) + display = base + suffix = 2 + while display in used_names: + display = f"{base} ({suffix})" + suffix += 1 + used_names.add(display) + display_by_id[str(node[0])] = display + + nodes = [ + (display_by_id.get(str(node_id), node_id), data) + for node_id, data in original_nodes + ] + edges = [] + support_aliases = {name: node_id for node_id, name in display_by_id.items()} + for edge in item.get("edges", []): + if not isinstance(edge, (list, tuple)) or len(edge) != 3: + continue + source, target, data = edge + display_source = display_by_id.get(str(source), source) + display_target = display_by_id.get(str(target), target) + edges.append((display_source, display_target, data)) + if str(source) in display_by_id: + support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" + elif str(target) in display_by_id: + support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" + + def resolve_support(support: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: + if not support: + return None + resolved = [] + for cited in support.get("cited", []): + if cited in support_aliases: + resolved.append(support_aliases[cited]) + else: + resolved.append(cited) + return {"cited": resolved} + + return { + "nodes": nodes, + "edges": edges, + "canonical_nodes": [str(node_id) for node_id, _ in original_nodes], + "resolve_support": resolve_support, + } def validate_support(support: Optional[dict[str, Any]], node_names: set[str]) -> Optional[dict[str, Any]]: - """cited 全部存在于分区节点集合 → 原样返回；否则返回 None（剥离）。""" + """Keep support only when all citations point to existing partition nodes.""" if not support: return None cited = support.get("cited") or [] diff --git a/graphgen/models/kg_builder/event_contract.py b/graphgen/models/kg_builder/event_contract.py index 374ea557..d6cf7350 100644 --- a/graphgen/models/kg_builder/event_contract.py +++ b/graphgen/models/kg_builder/event_contract.py @@ -1,10 +1,11 @@ -"""Pure stdlib contracts for SAG-adapted event extraction (testable without GraphGen).""" +"""Pure stdlib contracts for profile-driven SAG event extraction.""" from __future__ import annotations + import hashlib import json import re from collections import defaultdict -from typing import Any, Optional +from typing import Any PMS_ENTITY_TYPES: list[dict] = [ {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, @@ -16,36 +17,53 @@ {"type": "DATE", "description": "日期或时间"}, {"type": "METRIC", "description": "数量、比例等指标"}, ] - PROJECT_ID_RE = re.compile(r"\b[PI]-\d{6,}\b") _NUMBER_RE = re.compile(r"(? set[str]: - """原文/事件中的数字集合（SAG grounding 的确定性口径）。""" + """Return numeric strings using the deterministic SAG grounding policy.""" return set(_NUMBER_RE.findall(text or "")) -def event_anchor(content: str) -> str | None: - """事件的完整项目编号锚点；取首个出现的编号。""" - match = PROJECT_ID_RE.search(content or "") - return match.group(0) if match else None +def event_anchor(content: str, pattern: str | None = None) -> str | None: + """Return the first configured anchor match, if any.""" + if pattern is None: + return None + match = re.compile(pattern).search(content or "") + if match is None: + return None + if match.lastindex: + return match.group(1) + return match.group(0) -def event_node_name(anchor: str, title: str, content: str) -> str: - """确定性事件节点 id：event:{anchor}:{hash12}（内容 SHA，重叠 chunk 天然去重）。""" - digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] - return f"event:{anchor}:{digest}" +def event_node_name(anchor: str | None, title: str, content: str, *, legacy: bool = True) -> str: + """Create a deterministic node ID; legacy PMS mode preserves persisted IDs.""" + if legacy: + digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] + return f"event:{anchor}:{digest}" + digest = hashlib.sha256( + json.dumps([anchor, title, content], ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest()[:20] + return f"event:{digest}" -def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[list[dict], dict[str, int]]: - """解析并校验 LLM 事件抽取响应，返回 (有效事件列表, 丢弃计数)。 +def parse_extraction_response( + response: str, + *, + chunk_content: str, + entity_types: list[dict] | None = None, + anchor_pattern: str | None = None, + anchor_required: bool | None = None, + number_grounding: bool = True, +) -> tuple[list[dict], dict[str, int]]: + """Parse the strict event response contract under an explicit profile policy. - 严格合同（SAG schema.py 口径）：顶层/事件/实体禁合同外字段；is_valid=False、 - 无项目锚点、数字不接地、实体类型越界的事件整条丢弃并计数。 + Omitted policy arguments retain the historical PMS behavior for callers that + import this pure-stdlib helper directly. """ dropped: dict[str, int] = defaultdict(int) text = (response or "").strip() @@ -59,22 +77,32 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis except json.JSONDecodeError: dropped["response_not_json"] += 1 return [], dict(dropped) - if not isinstance(payload, dict) or set(payload.keys()) != {"type", "data"}: + if not isinstance(payload, dict) or set(payload) != {"type", "data"}: dropped["response_contract_invalid"] += 1 return [], dict(dropped) data = payload.get("data") - if not isinstance(data, dict) or set(data.keys()) != {"items"}: + if not isinstance(data, dict) or set(data) != {"items"} or not isinstance(data["items"], list): dropped["response_contract_invalid"] += 1 return [], dict(dropped) - evidence_numbers = numbers_in(chunk_content) + profile_policy = entity_types is not None or anchor_pattern is not None or anchor_required is not None + if profile_policy: + types = entity_types or [] + allowed_types = {str(item["type"]).strip().upper() for item in types} + if anchor_required is None: + anchor_required = True + else: + types = PMS_ENTITY_TYPES + allowed_types = {item["type"] for item in types} + anchor_pattern = PROJECT_ID_RE.pattern + anchor_required = True + evidence_numbers = numbers_in(chunk_content) if number_grounding else set() valid_events: list[dict] = [] - for item in data.get("items") or []: + for item in data["items"]: if not isinstance(item, dict): dropped["event_not_object"] += 1 continue - extra_keys = set(item.keys()) - _ALLOWED_EVENT_KEYS - if extra_keys: + if set(item) - _ALLOWED_EVENT_KEYS: dropped["event_extra_fields"] += 1 continue title = str(item.get("title") or "").strip() @@ -86,20 +114,22 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis if item.get("is_valid") is False: dropped["event_self_invalid"] += 1 continue - anchor = event_anchor(content) - if anchor is None: + anchor = event_anchor(content, anchor_pattern) + if anchor_required and anchor is None: dropped["event_missing_anchor"] += 1 continue - ungrounded = {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers} - if ungrounded: + if number_grounding and {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers}: dropped["event_number_not_grounded"] += 1 continue entities: list[dict] = [] + if not isinstance(entities_raw, list): + dropped["event_entities_invalid"] += 1 + continue for entity in entities_raw: if not isinstance(entity, dict): dropped["entity_not_object"] += 1 continue - if set(entity.keys()) - _ALLOWED_ENTITY_KEYS: + if set(entity) - _ALLOWED_ENTITY_KEYS: dropped["entity_extra_fields"] += 1 continue etype = str(entity.get("type") or "").strip().upper() @@ -108,12 +138,9 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis if not etype or not ename or not edesc: dropped["entity_blank"] += 1 continue - if etype not in _ALLOWED_TYPES: + if etype not in allowed_types: dropped["entity_type_unknown"] += 1 continue entities.append({"type": etype, "name": ename, "description": edesc}) - valid_events.append( - {"title": title, "content": content, "anchor": anchor, "entities": entities} - ) + valid_events.append({"title": title, "content": content, "anchor": anchor, "entities": entities}) return valid_events, dict(dropped) - diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py index 1aa44489..f7dfee7f 100644 --- a/graphgen/models/kg_builder/event_entity_kg_builder.py +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -1,54 +1,33 @@ -"""Event-entity KG builder (PMS fork, SAG-adapted). +"""Profile-driven SAG-adapted event/entity graph builder. -来源与许可：核心设计改编自 Zleap-AI/SAG `zleap/sag/modules/extract/{schema,grounding}.py` -与 `prompts/extract_document.yaml` v3.1（MIT，Copyright (c) 2026 sag contributors， -arXiv:2606.15971）。改编点见 PMS fork docs/PMS_PATCHES.md #7 与 Phase 2 方案文档 §3.2。 - -与 LightRAGKGBuilder 的差异： -- 每 chunk 抽取"语义完整的事件"而非三元组：事件 = EVENT 节点 + "事件→参与者" - 星型边（超边的星型编码，n 元关系不拆三元组）； -- 严格响应合同（禁合同外字段、is_valid 自报、无锚 fail-closed 丢弃）； -- 数字接地（SAG grounding 移植）：事件中一切数字必须在原文逐字出现，否则弃； -- 事件节点 id 确定性（内容 SHA），chunk 重叠去重由官方 merge_nodes SEP 合并免费获得。 - -extract() 返回与官方 ({entity_name: [node]}, {(src, tgt): [edge]}) 相同的形状， -merge_nodes / merge_edges / 描述摘要全部复用官方实现。 +Domain policy (prompt, entity types, anchor rules and grounding) comes from a +validated event extraction profile. Generic graph construction remains shared. +SAG-derived adaptation and license notes are recorded in docs/PMS_PATCHES.md. """ from __future__ import annotations import json - from collections import defaultdict -from typing import Any, Dict, List, Tuple +from typing import Dict, List, Tuple from graphgen.bases import BaseGraphStorage, BaseLLMWrapper, Chunk -from graphgen.templates import EVENT_ENTITY_EXTRACTION_PROMPT -from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES -from graphgen.utils import detect_main_language, logger +from graphgen.models.kg_builder.event_contract import event_node_name, parse_extraction_response +from graphgen.templates.event_profile_loader import EventExtractionProfile, load_event_profile +from graphgen.utils import logger _EVENT_DESC_CAP = 4000 _ENTITY_DESC_CAP = 2000 -from graphgen.models.kg_builder.event_contract import ( - PROJECT_ID_RE, - _ALLOWED_ENTITY_KEYS, - _ALLOWED_EVENT_KEYS, - _ALLOWED_TYPES, - event_anchor, - event_node_name, - numbers_in, - parse_extraction_response, -) - class EventEntityKGBuilder: """事件-实体抽取器：extract → 星型编码节点/边；合并复用官方 LightRAG merge。""" - def __init__(self, llm_client: BaseLLMWrapper): + def __init__(self, llm_client: BaseLLMWrapper, profile: str | EventExtractionProfile | None = None): from graphgen.models import LightRAGKGBuilder self.llm_client = llm_client + self.profile = load_event_profile(profile) if isinstance(profile, (str, type(None))) else profile self._merger = LightRAGKGBuilder(llm_client=llm_client) async def extract( @@ -56,8 +35,7 @@ async def extract( ) -> Tuple[Dict[str, List[dict]], Dict[Tuple[str, str], List[dict]]]: """单 chunk 事件抽取，返回与官方同形的 (nodes_data, edges_data)。""" content = chunk.content - language = detect_main_language(content) - entity_types_json = json.dumps(PMS_ENTITY_TYPES, ensure_ascii=False) + entity_types_json = json.dumps(list(self.profile.entity_types), ensure_ascii=False) request_json = json.dumps( { "type": "request", @@ -68,11 +46,17 @@ async def extract( }, ensure_ascii=False, ).replace("__ENTITY_TYPES__", entity_types_json) - # 模板含 JSON 字面大括号，禁用 str.format；用占位符替换注入输入。 - prompt = EVENT_ENTITY_EXTRACTION_PROMPT[language].replace("{input_text}", request_json) + prompt = self.profile.prompt.replace("{input_text}", request_json) response = await self.llm_client.generate_answer(prompt) - events, dropped = parse_extraction_response(response, chunk_content=content) + events, dropped = parse_extraction_response( + response, + chunk_content=content, + entity_types=list(self.profile.entity_types), + anchor_pattern=self.profile.anchor_pattern, + anchor_required=self.profile.anchor_required, + number_grounding=self.profile.number_grounding, + ) if sum(dropped.values()): logger.warning( "Event extraction dropped: %s (chunk=%s)", dropped, chunk.id @@ -84,30 +68,34 @@ async def extract( edges: Dict[Tuple[str, str], List[dict]] = defaultdict(list) for event in events: anchor = event["anchor"] - event_name = event_node_name(anchor, event["title"], event["content"]) + event_name = event_node_name( + anchor, event["title"], event["content"], legacy=self.profile.legacy_event_id + ) description = f"{event['title']}：{event['content']}" - # 事件节点（EVENT 实体，扁平属性；参与者由星型边表达） + # 事件节点使用通用节点属性；分区器读取 anchor，不解析节点 ID。 nodes[event_name].append( { - "entity_type": "EVENT", + "entity_type": self.profile.event_entity_type, "entity_name": event_name, + "graph_role": "event", "description": description, + "anchor": anchor or "", "source_id": chunk.id, } ) - # 参与者实体（去重保序；锚点项目实体若缺失则补齐） + # Add the configured anchor entity only when the profile requests one. seen: set[str] = set() participants: list[dict] = [] for entity in event["entities"]: if entity["name"] not in seen: seen.add(entity["name"]) participants.append(entity) - if anchor not in seen: + if anchor is not None and self.profile.anchor_entity_type and anchor not in seen: participants.append( { - "type": "PROJECT", + "type": self.profile.anchor_entity_type, "name": anchor, - "description": f"项目 {anchor}，本事项的锚定项目", + "description": f"锚点 {anchor}，本事项的分组标识", } ) seen.add(anchor) @@ -133,7 +121,7 @@ async def extract( async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: entity_name, node_list = node_data - if node_list and str(node_list[0].get("entity_type", "")).upper() == "EVENT": + if node_list and str(node_list[0].get("entity_type", "")).upper() == self.profile.event_entity_type: return await self._merge_event_node(entity_name, node_list, kg_instance) # 星型事件参与者是图投影，不需要再次 LLM 摘要；只做确定性去重合并， # 避免 LightRAG summarizer 把跨事件上下文扩写成伪业务事实。 @@ -166,13 +154,20 @@ async def _merge_event_node( ) -> dict: """事件节点确定性合并：SEP 去重拼接（截断），不走 LLM 摘要—— 官方摘要器会把事件 id 当实体名写进描述，污染出题上下文。""" + existing = kg_instance.get_node(entity_name) or {} + anchors = {str(dp.get("anchor") or "") for dp in node_list if dp.get("anchor")} + existing_anchor = str(existing.get("anchor") or "") + if existing_anchor: + anchors.add(existing_anchor) merged = "".join(sorted({dp["description"] for dp in node_list})) if len(merged) > _EVENT_DESC_CAP: merged = merged[:_EVENT_DESC_CAP] node_data_dict = { - "entity_type": "EVENT", + "entity_type": self.profile.event_entity_type, "entity_name": entity_name, + "graph_role": "event", "description": merged, + "anchor": "".join(sorted(anchors)), "source_id": "".join( sorted({sid for dp in node_list for sid in str(dp.get("source_id", "")).split("") if sid} | {sid for sid in str((kg_instance.get_node(entity_name) or {}).get("source_id", "")).split("") if sid}) @@ -184,23 +179,31 @@ async def _merge_event_node( async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: (src_id, tgt_id), edge_list = edges_data - if src_id.startswith("event:") or tgt_id.startswith("event:"): - # 事件成员边同样确定性合并，不做 LLM 摘要 + event_ids = { + str(dp["src_id"]) + for dp in edge_list + if dp.get("event_member") and dp.get("src_id") + } + if src_id in event_ids or tgt_id in event_ids: + # 事件成员边同样确定性合并，不做 LLM 摘要。 + event_id = src_id if src_id in event_ids else tgt_id + participant_id = tgt_id if event_id == src_id else src_id merged = "".join(sorted({dp["description"] for dp in edge_list})) if len(merged) > _EVENT_DESC_CAP: merged = merged[:_EVENT_DESC_CAP] + existing_edge = kg_instance.get_edge(src_id, tgt_id) or kg_instance.get_edge(tgt_id, src_id) or {} source_ids = { sid for dp in edge_list for sid in str(dp.get("source_id", "")).split("") if sid } | { - sid for sid in str((kg_instance.get_edge(src_id, tgt_id) or {}).get("source_id", "")).split("") if sid + sid for sid in str(existing_edge.get("source_id", "")).split("") if sid } edge_data = { - "src_id": src_id, - "tgt_id": tgt_id, + "src_id": event_id, + "tgt_id": participant_id, "description": merged, "source_id": "".join(sorted(source_ids)), "length": self._merger.tokenizer.count_tokens(merged), } - kg_instance.upsert_edge(src_id, tgt_id, edge_data=edge_data) + kg_instance.upsert_edge(event_id, participant_id, edge_data=edge_data) return edge_data return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py index 3414ff16..327e447d 100644 --- a/graphgen/models/partitioner/event_join_partitioner.py +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -1,27 +1,15 @@ -"""Event-join partitioner (PMS fork, SAG join semantics). +"""Generic shared-participant event join partitioner (SAG join semantics). -来源：共享实体 join 语义翻译自 Zleap-AI/SAG `modules/search/base.py` 的 -entity→event→relation 两跳查询（MIT，arXiv:2606.15971）；离线物化见 Phase 2 方案文档。 - -事件 = EVENT 节点（id 形如 ``event:{anchor}:{hash12}``）+ 星型成员边。分区算法： -1. 种子事件按节点 id 排序（确定性）； -2. 通过共享参与者实体 join 一跳邻居事件； -3. 硬约束：只接受同 project_anchor 事件（簇项目纯度 100%）； -4. 容量上限：max_events_per_community 或事件描述 token 总和。 - -产出官方 ``Community``（nodes = 事件+参与者实体，edges = 星型成员边）， -metadata 携带 event_ids 供出题端 support 校验。 +Anchor isolation reads an explicit node attribute and never infers domain data +from event ID strings. """ -import math from collections import deque from typing import Any, Iterable, List, Optional, Set, Tuple from graphgen.bases import BaseGraphStorage, BasePartitioner from graphgen.bases.datatypes import Community -EVENT_NODE_PREFIX = "event:" - class EventJoinPartitioner(BasePartitioner): """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" @@ -32,13 +20,24 @@ def partition( max_events_per_community: int = 6, max_tokens_per_community: int = 4096, min_events_per_community: int = 1, + event_entity_type: str = "EVENT", + anchor_attribute: str = "anchor", + require_same_anchor: bool = True, **kwargs: Any, ) -> Iterable[Community]: + if not isinstance(require_same_anchor, bool): + raise ValueError("event_join_require_same_anchor_must_be_bool") + if not isinstance(event_entity_type, str) or not event_entity_type.strip(): + raise ValueError("event_join_event_entity_type_required") + if not isinstance(anchor_attribute, str) or not anchor_attribute.strip(): + raise ValueError("event_join_anchor_attribute_required") + if max_events_per_community < 1 or min_events_per_community < 1 or max_tokens_per_community < 1: + raise ValueError("event_join_limits_must_be_positive") events: List[Tuple[str, dict]] = [] node_dict: dict[str, dict] = {} for nid, data in g.get_all_nodes(): node_dict[nid] = data - if str(data.get("entity_type", "")).upper() == "EVENT": + if str(data.get("entity_type", "")).upper() == event_entity_type.upper(): events.append((nid, data)) events.sort(key=lambda item: item[0]) @@ -54,8 +53,7 @@ def partition( used: Set[str] = set() def _anchor(event_id: str) -> str: - # event:{anchor}:{hash} - return event_id[len(EVENT_NODE_PREFIX):].rsplit(":", 1)[0] + return str(node_dict[event_id].get(anchor_attribute, "")) def _grow(seed: Tuple[str, dict]) -> Optional[Community]: member_events: List[str] = [] @@ -64,6 +62,8 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: member_edges: Set[frozenset] = set() token_sum = 0 anchor = _anchor(seed[0]) + if require_same_anchor and not anchor: + raise ValueError(f"event_join_anchor_missing:{seed[0]}") queue = deque([seed[0]]) while queue: @@ -87,7 +87,7 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: for other in events_by_entity.get(entity_id, []): if other in used or other in event_set: continue - if _anchor(other) != anchor: + if require_same_anchor and _anchor(other) != anchor: continue if token_sum + int(node_dict[other].get("length", 0)) > max_tokens_per_community: continue diff --git a/graphgen/operators/build_kg/build_event_kg.py b/graphgen/operators/build_kg/build_event_kg.py index a25ca139..95c42236 100644 --- a/graphgen/operators/build_kg/build_event_kg.py +++ b/graphgen/operators/build_kg/build_event_kg.py @@ -1,4 +1,4 @@ -"""Event-entity KG construction over text chunks (PMS fork).""" +"""Construct an event/entity star graph from text chunks.""" from __future__ import annotations @@ -16,9 +16,10 @@ def build_event_entity_kg( llm_client: BaseLLMWrapper, kg_instance: BaseGraphStorage, chunks: List[Chunk], + event_profile: str | None = None, ) -> tuple: """chunks → 事件星型图（合并/摘要/SEP 血缘全部复用官方 LightRAG merge）。""" - kg_builder = EventEntityKGBuilder(llm_client=llm_client) + kg_builder = EventEntityKGBuilder(llm_client=llm_client, profile=event_profile) results = run_concurrent( kg_builder.extract, @@ -30,11 +31,21 @@ def build_event_entity_kg( nodes = defaultdict(list) edges = defaultdict(list) + event_ids = { + event_id + for node_map, _ in results + for event_id, rows in node_map.items() + if rows and str(rows[0].get("entity_type", "")).upper() == kg_builder.profile.event_entity_type + } for n, e in results: for k, v in n.items(): nodes[k].extend(v) for k, v in e.items(): - edges[tuple(sorted(k))].extend(v) + normalized_key = tuple(sorted(k)) + if any(endpoint in event_ids for endpoint in normalized_key): + for edge in v: + edge["event_member"] = True + edges[normalized_key].extend(v) nodes = run_concurrent( lambda kv: kg_builder.merge_nodes(kv, kg_instance=kg_instance), diff --git a/graphgen/operators/build_kg/build_kg_service.py b/graphgen/operators/build_kg/build_kg_service.py index 9a3c484c..c86c8657 100644 --- a/graphgen/operators/build_kg/build_kg_service.py +++ b/graphgen/operators/build_kg/build_kg_service.py @@ -28,9 +28,13 @@ def __init__( ) self.build_kwargs = build_kwargs self.max_loop: int = int(self.build_kwargs.get("max_loop", 3)) - # PMS fork（Phase 2）：文本 KG 抽取方法分发。默认 light_rag 与官方逐字节 - # 一致；event_entity 走事件-实体星型编码（见 models/kg_builder/event_entity_kg_builder.py）。 + # Opt-in fork capability. Existing configs continue to use LightRAG by default. self.kg_method: str = str(self.build_kwargs.get("kg_method", "light_rag")) + self.event_profile = self.build_kwargs.get("event_profile") + if self.kg_method == "event_entity": + from graphgen.templates.event_profile_loader import load_event_profile + + self.event_profile = load_event_profile(self.event_profile).profile_id def process(self, batch: list) -> Tuple[list, dict]: """ @@ -59,6 +63,7 @@ def process(self, batch: list) -> Tuple[list, dict]: llm_client=self.llm_client, kg_instance=self.graph_storage, chunks=text_chunks, + event_profile=self.event_profile, ) nodes += text_nodes edges += text_edges diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index e6676fe6..5f35f2ab 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -122,7 +122,13 @@ def process(self, batch: list) -> Tuple[list, dict]: Generate question-answer pairs based on nodes and edges. """ logger.info("[Generation] mode: %s, batches: %d", self.method, len(batch)) - triples = [(item["nodes"], item["edges"]) for item in batch] + from graphgen.models.generator.support import build_generation_view, validate_support + + generation_views = [build_generation_view(item) for item in batch] + triples = [ + (view["nodes"], view["edges"]) + for view in generation_views + ] results = run_concurrent( self.generator.generate, triples, @@ -134,19 +140,13 @@ def process(self, batch: list) -> Tuple[list, dict]: meta_updates = {} final_results = [] - for item, qa_pairs in zip(batch, results): + for item, view, qa_pairs in zip(batch, generation_views, results): if not qa_pairs: continue - # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 - # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 - node_names = { - node[0] - for node in item.get("nodes", []) - if isinstance(node, (list, tuple)) and node - } for qa_pair in qa_pairs: support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None - support = validate_support(support, node_names) + support = view["resolve_support"](support) + support = validate_support(support, set(view["canonical_nodes"])) if support: qa_pair["support"] = support res = self.generator.format_generation_results( diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index db355769..cd1c1dca 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -29,6 +29,11 @@ def __init__( self.tokenizer_instance: BaseTokenizer = Tokenizer(model_name=tokenizer_model) method = partition_kwargs["method"] self.method_params = partition_kwargs.get("method_params", {}) + if method == "event_join": + self.method_params = dict(self.method_params) + self.method_params.setdefault("event_entity_type", partition_kwargs.get("event_entity_type", "EVENT")) + self.method_params.setdefault("anchor_attribute", partition_kwargs.get("anchor_attribute", "anchor")) + self.method_params.setdefault("require_same_anchor", partition_kwargs.get("require_same_anchor", True)) if method == "bfs": from graphgen.models import BFSPartitioner diff --git a/graphgen/templates/event_profile_loader.py b/graphgen/templates/event_profile_loader.py new file mode 100644 index 00000000..c5a502a3 --- /dev/null +++ b/graphgen/templates/event_profile_loader.py @@ -0,0 +1,129 @@ +"""Load and validate packaged event extraction profiles.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any + +PROFILE_SCHEMA = "graphgen_event_profile.v1" +PROFILE_ROOT = files("graphgen.templates").joinpath("event_profiles") + + +@dataclass(frozen=True) +class EventExtractionProfile: + profile_id: str + entity_types: tuple[dict[str, str], ...] + prompt: str + anchor_pattern: str | None + anchor_required: bool + number_grounding: bool + event_entity_type: str + anchor_entity_type: str | None + legacy_event_id: bool + manifest_sha256: str + prompt_sha256: str + + +def _load_prompt_resource(profile_id: str, prompt_file: str) -> bytes: + if not prompt_file or Path(prompt_file).name != prompt_file: + raise ValueError("event_profile_prompt_file_invalid") + return PROFILE_ROOT.joinpath(profile_id, prompt_file).read_bytes() + + +def load_event_profile(profile: str | None = None) -> EventExtractionProfile: + """Load a built-in profile ID or explicit manifest path; fail closed.""" + profile_id = "pms_event.v1" if profile is None else profile + if not isinstance(profile_id, str) or not profile_id.strip(): + raise ValueError("event_profile_must_be_non_empty_string") + if not "/" in profile_id and not "\\" in profile_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", profile_id): + raise ValueError("event_profile_id_invalid") + if "/" in profile_id or "\\" in profile_id: + manifest_path = Path(profile_id).resolve() + if not manifest_path.is_file(): + raise ValueError(f"event_profile_not_found:{profile_id}") + manifest_bytes = manifest_path.read_bytes() + data = json.loads(manifest_bytes.decode("utf-8")) + prompt_file = str(data.get("prompt_file", "")) + if not prompt_file or Path(prompt_file).name != prompt_file: + raise ValueError("event_profile_prompt_file_invalid") + prompt_path = (manifest_path.parent / prompt_file).resolve() + if prompt_path.parent != manifest_path.parent: + raise ValueError("event_profile_prompt_file_invalid") + prompt_bytes = prompt_path.read_bytes() + else: + manifest_resource = PROFILE_ROOT.joinpath(profile_id, "profile.json") + if not manifest_resource.is_file(): + raise ValueError(f"event_profile_not_found:{profile_id}") + manifest_bytes = manifest_resource.read_bytes() + data = json.loads(manifest_bytes.decode("utf-8")) + prompt_bytes = _load_prompt_resource(profile_id, str(data.get("prompt_file", ""))) + + if not isinstance(data, dict) or data.get("schema_version") != PROFILE_SCHEMA: + raise ValueError("event_profile_schema_unsupported") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", str(data.get("profile_id", ""))): + raise ValueError("event_profile_id_invalid") + if data.get("profile_id") != profile_id and "/" not in profile_id and "\\" not in profile_id: + raise ValueError("event_profile_id_mismatch") + entity_types = data.get("entity_types") + if not isinstance(entity_types, list) or not entity_types: + raise ValueError("event_profile_entity_types_required") + normalized: list[dict[str, str]] = [] + seen: set[str] = set() + for item in entity_types: + if not isinstance(item, dict) or set(item) != {"type", "description"}: + raise ValueError("event_profile_entity_type_invalid") + name = str(item["type"]).strip().upper() + description = str(item["description"]).strip() + if not re.fullmatch(r"[A-Z][A-Z0-9_]{0,63}", name) or not description or name in seen: + raise ValueError("event_profile_entity_type_invalid") + seen.add(name) + normalized.append({"type": name, "description": description}) + + anchor = data.get("anchor") + if not isinstance(anchor, dict) or set(anchor) != {"required", "pattern", "entity_type"}: + raise ValueError("event_profile_anchor_invalid") + required, pattern, anchor_entity_type = anchor["required"], anchor["pattern"], anchor["entity_type"] + if not isinstance(required, bool) or (pattern is not None and not isinstance(pattern, str)): + raise ValueError("event_profile_anchor_invalid") + if required and not pattern: + raise ValueError("event_profile_required_anchor_pattern_missing") + if pattern: + try: + re.compile(pattern) + except re.error as error: + raise ValueError("event_profile_anchor_pattern_invalid") from error + if anchor_entity_type is not None and (not isinstance(anchor_entity_type, str) or anchor_entity_type.upper() not in seen): + raise ValueError("event_profile_anchor_entity_type_unknown") + if isinstance(anchor_entity_type, str): + anchor_entity_type = anchor_entity_type.upper() + + grounding = data.get("grounding") + if not isinstance(grounding, dict) or set(grounding) != {"number_grounding"} or not isinstance(grounding["number_grounding"], bool): + raise ValueError("event_profile_grounding_invalid") + graph = data.get("graph") + if not isinstance(graph, dict) or set(graph) != {"event_entity_type", "legacy_event_id"}: + raise ValueError("event_profile_graph_invalid") + event_entity_type = str(graph["event_entity_type"]).strip().upper() + if not re.fullmatch(r"[A-Z][A-Z0-9_]{0,63}", event_entity_type) or not isinstance(graph["legacy_event_id"], bool): + raise ValueError("event_profile_event_entity_type_invalid") + prompt = prompt_bytes.decode("utf-8") + if not prompt.strip() or prompt.count("{input_text}") != 1: + raise ValueError("event_profile_prompt_invalid") + + return EventExtractionProfile( + profile_id=str(data["profile_id"]), + entity_types=tuple(normalized), + prompt=prompt, + anchor_pattern=pattern, + anchor_required=required, + number_grounding=grounding["number_grounding"], + event_entity_type=event_entity_type, + anchor_entity_type=anchor_entity_type, + legacy_event_id=graph["legacy_event_id"], + manifest_sha256=hashlib.sha256(manifest_bytes).hexdigest(), + prompt_sha256=hashlib.sha256(prompt_bytes).hexdigest(), + ) diff --git a/graphgen/templates/event_profiles/example_event.v1/profile.json b/graphgen/templates/event_profiles/example_event.v1/profile.json new file mode 100644 index 00000000..95fe1403 --- /dev/null +++ b/graphgen/templates/event_profiles/example_event.v1/profile.json @@ -0,0 +1,13 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "example_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "MISSION", "description": "一次需要完成的任务或行动"}, + {"type": "ACTOR", "description": "参与任务的个人或组织"}, + {"type": "RESOURCE", "description": "任务使用或影响的资源"} + ], + "anchor": {"required": true, "pattern": "\\bCASE-[A-Z0-9-]+\\b", "entity_type": null}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": false} +} diff --git a/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt new file mode 100644 index 00000000..5d6ec7c2 --- /dev/null +++ b/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt @@ -0,0 +1,28 @@ +## Role + +你是一个通用的事件与参与实体抽取器。只依据当前文本片段，识别可独立陈述的事件及其参与者。 + +## Rules + +- 每个事件以完整、可读的事实句表达，区分已经发生、正在发生、计划和推测。 +- 不补充原文未支持的常识、背景或因果结论；信息不足时不输出该事件。 +- 只使用输入 meta.entity_types 中声明的实体类型，不自造类型。 +- 实体 description 说明其在该事件中的作用；并列参与者分别列出。 +- 如果输入文本存在 profile 配置的锚点，事件 content 应保留该锚点；不得自行创造锚点。 +- 所有数字必须可在当前输入原文中逐字找到。 +- 相对时间保留原文，不自行换算。 + +## Input + +输入 JSON：{"type":"request","data":{"items":[{"id":1,"content":"..."}],"meta":{"entity_types":[{"type":"...","description":"..."}]}}} + +## Output + +只返回严格 JSON，不得包含 Markdown 或其他文字： +{"type":"response","data":{"items":[{"title":"...","content":"...","entities":[{"type":"...","name":"...","description":"..."}],"is_valid":true}]}} +无有效事件时返回：{"type":"response","data":{"items":[]}} + +给定输入： +{input_text} + +输出： diff --git a/graphgen/templates/event_profiles/generic_event.v1/profile.json b/graphgen/templates/event_profiles/generic_event.v1/profile.json new file mode 100644 index 00000000..b6155c66 --- /dev/null +++ b/graphgen/templates/event_profiles/generic_event.v1/profile.json @@ -0,0 +1,12 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "generic_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "ACTION", "description": "An action or occurrence"}, + {"type": "ACTOR", "description": "A participant in the event"} + ], + "anchor": {"required": false, "pattern": null, "entity_type": null}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": false} +} diff --git a/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt new file mode 100644 index 00000000..b7d9b189 --- /dev/null +++ b/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt @@ -0,0 +1,27 @@ +## Role + +You extract events and participants from the supplied text. Report only facts supported by the input. + +## Rules + +- Write each event as a complete, independently readable factual statement. +- Distinguish completed, ongoing, planned, and uncertain actions. +- Use only entity types listed in `meta.entity_types`; do not invent types. +- Describe each participant's role in the event and split distinct participants. +- Preserve numbers and dates only when supported by the source. Keep relative dates as written. +- Return no event when the source does not support a useful standalone fact. + +## Input + +The input is JSON with `type=request`, `data.items` and `data.meta.entity_types`. + +## Output + +Return only JSON matching this exact shape: +{"type":"response","data":{"items":[{"title":"...","content":"...","entities":[{"type":"...","name":"...","description":"..."}],"is_valid":true}]}} +For no events return `{"type":"response","data":{"items":[]}}`. + +Input: +{input_text} + +Output: diff --git a/graphgen/templates/event_profiles/pms_event.v1/profile.json b/graphgen/templates/event_profiles/pms_event.v1/profile.json new file mode 100644 index 00000000..5a39839e --- /dev/null +++ b/graphgen/templates/event_profiles/pms_event.v1/profile.json @@ -0,0 +1,18 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "pms_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"} + ], + "anchor": {"required": true, "pattern": "\\b[PI]-\\d{6,}\\b", "entity_type": "PROJECT"}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": true} +} diff --git a/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt new file mode 100644 index 00000000..7ae0749f --- /dev/null +++ b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt @@ -0,0 +1,74 @@ +## Role + +你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 +构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 + +## 事项提取原则 + +- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 +- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， + 保留必要的数字和结论。 +- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 +- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 +- 不得把外部常识、文件名或模型推测写成原文事实。 +- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 + （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 + 编号的事项不得输出。 +- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 +- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 + 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 +- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， + 不得写成"某人属于哪个部门"等组织反查事实。 + +## 实体提取原则 + +- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， + 没有合适类型时不要自造类型。 +- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 +- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 +- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 +- 同名实体保持一致；不同实体不得因名称相近而合并。 + +## 事实边界（fail-closed） + +- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 +- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； + 找不到就删掉该表述或放弃该事项。 +- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 + +## Task + +按以下顺序执行，不能跳步： + +1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； + 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 +2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > + 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 +3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 + 编号锚点、禁止推理说明。 +4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 +5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； + 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 + 全部片段为噪音或信息不足时返回 items: []。 + +## Input + +输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], +"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} + +## Output requirements + +- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： + {"type": "response", "data": {"items": [ + {"title": "...", "content": "...", + "entities": [{"type": "...", "name": "...", "description": "..."}], + "is_valid": true} + ]}} +- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null + 或制造占位事项。 +- 相对时间（如"下月"）保留原文，不自行换算。 + +给定输入： +{input_text} + +输出： diff --git a/graphgen/templates/kg/event_entity_extraction.py b/graphgen/templates/kg/event_entity_extraction.py index e74c1be8..666f9f8d 100644 --- a/graphgen/templates/kg/event_entity_extraction.py +++ b/graphgen/templates/kg/event_entity_extraction.py @@ -1,113 +1,17 @@ -# pylint: disable=C0301 -"""事件-实体抽取提示词（SAG extract_document v3.1 改编，MIT）。 +"""Backward-compatible exports for the original PMS event extraction prompt.""" +from __future__ import annotations -来源：Zleap-AI/SAG `zleap/sag/prompts/extract_document.yaml`（arXiv 2606.15971）。 -改编点（PMS project-query 场景）： -- 事件必须含完整项目编号锚点（P-/I- + ≥6 位数字），无锚事件直接不产出； -- 字段锁定 / 反查禁止 / 越界语义禁止（与 pms_policy 出题铁律同源）; -- 数字接地：事件中一切数字必须在原文出现； -- 只输出扁平事项（不用 children 层级），便于星型编码进图。 -响应是严格 JSON 合同：禁合同外字段，无效字段/无锚/数字不接地一律丢弃并计数。 -""" - -TEMPLATE_ZH: str = """## Role - -你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 -构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 - -## 事项提取原则 - -- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 -- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， - 保留必要的数字和结论。 -- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 -- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 -- 不得把外部常识、文件名或模型推测写成原文事实。 -- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 - （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 - 编号的事项不得输出。 -- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 -- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 - 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 -- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， - 不得写成"某人属于哪个部门"等组织反查事实。 - -## 实体提取原则 - -- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， - 没有合适类型时不要自造类型。 -- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 -- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 -- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 -- 同名实体保持一致；不同实体不得因名称相近而合并。 - -## 事实边界（fail-closed） - -- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 -- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； - 找不到就删掉该表述或放弃该事项。 -- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 - -## Task - -按以下顺序执行，不能跳步： - -1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； - 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 -2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > - 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 -3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 - 编号锚点、禁止推理说明。 -4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 -5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； - 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 - 全部片段为噪音或信息不足时返回 items: []。 - -## Input - -输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], -"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} - -## Output requirements - -- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： - {"type": "response", "data": {"items": [ - {"title": "...", "content": "...", - "entities": [{"type": "...", "name": "...", "description": "..."}], - "is_valid": true} - ]}} -- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null - 或制造占位事项。 -- 相对时间（如"下月"）保留原文，不自行换算。 - -给定输入： -{input_text} - -输出： -""" - -#: 紧凑 JSON 请求包装（与 SAG request 合同同形；PMS 单 chunk 单 items） -INPUT_FORMAT: dict = { - "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' - '"meta": {{"entity_types": {entity_types}}}}}}}', -} - -#: PMS 实体类型集（SAG 5W1H 类型集收敛到 project-query 可查询维度） -PMS_ENTITY_TYPES: list[dict] = [ - {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, - {"type": "PERSON", "description": "人员，项目成员或干系人"}, - {"type": "ROLE", "description": "项目内职务或角色，如二级项目经理、销售工程师"}, - {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, - {"type": "PHASE", "description": "项目阶段或状态，如售前、执行"}, - {"type": "LOCATION", "description": "地点或区域"}, - {"type": "DATE", "description": "日期或时间"}, - {"type": "METRIC", "description": "数量、比例等指标"}, -] +from graphgen.templates.event_profile_loader import load_event_profile +_pms_profile = load_event_profile("pms_event.v1") +PMS_ENTITY_TYPES: list[dict] = list(_pms_profile.entity_types) +TEMPLATE_ZH = _pms_profile.prompt EVENT_ENTITY_EXTRACTION_PROMPT = { "en": TEMPLATE_ZH, "zh": TEMPLATE_ZH, - "FORMAT": { - "entity_types": None, # 运行时注入 PMS_ENTITY_TYPES - }, + "FORMAT": {"entity_types": None}, +} +INPUT_FORMAT = { + "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' + '"meta": {{"entity_types": {entity_types}}}}}}}', } diff --git a/requirements-dev.txt b/requirements-dev.txt index 55b033e9..e079f8a6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -pytest \ No newline at end of file +pytest diff --git a/setup.py b/setup.py index 2b30f103..bc2b362c 100644 --- a/setup.py +++ b/setup.py @@ -38,14 +38,20 @@ def read_requirements(): setup( name="graphg", version=get_version(), - url="https://github.com/open-sciencelab/GraphGen", + url="https://github.com/Leon-Algo/GraphGen", description="GraphGen: Enhancing Supervised Fine-Tuning for LLMs with Knowledge-Driven Synthetic Data Generation", long_description=readme(), long_description_content_type="text/markdown", - author="open-sciencelab", - author_email="open-sciencelab@pjlab.org.cn", + author="Leon-Algo GraphGen maintainers", + author_email="", packages=find_packages(exclude=["models"]), - package_data={"GraphGen": ["configs/*"]}, + package_data={ + "graphgen": [ + "templates/profiles/*/*.json", + "templates/event_profiles/*/*.json", + "templates/event_profiles/*/*.txt", + ], + }, include_package_data=True, install_requires=install_packages, classifiers=[ diff --git a/tests/integration_tests/models/partitioner/test_bfs_partitioner.py b/tests/integration_tests/models/partitioner/test_bfs_partitioner.py index 48558cc1..bf6840ef 100644 --- a/tests/integration_tests/models/partitioner/test_bfs_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_bfs_partitioner.py @@ -1,43 +1,39 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import BFSPartitioner, NetworkXStorage +from graphgen.models import BFSPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_empty_graph(): +def test_empty_graph(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="empty") partitioner = BFSPartitioner() - communities = await partitioner.partition(storage, max_units_per_community=5) + communities = list(partitioner.partition(storage, max_units_per_community=5)) assert communities == [] -@pytest.mark.asyncio -async def test_single_node(): +def test_single_node(): nodes = [("A", {"desc": "alone"})] edges = [] with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="single_node") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = BFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5 - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_small_graph(): +def test_small_graph(): """ 0 - 1 - 2 | | | @@ -59,14 +55,14 @@ async def test_small_graph(): storage = NetworkXStorage(working_dir=tmpdir, namespace="small_graph") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = BFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4 - ) + )) assert len(communities) <= 5 diff --git a/tests/integration_tests/models/partitioner/test_dfs_partitioner.py b/tests/integration_tests/models/partitioner/test_dfs_partitioner.py index 536e139a..0176910c 100644 --- a/tests/integration_tests/models/partitioner/test_dfs_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_dfs_partitioner.py @@ -1,25 +1,22 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import DFSPartitioner, NetworkXStorage +from graphgen.models import DFSPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_empty_graph(): +def test_empty_graph(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage( working_dir=tmpdir, namespace="empty", ) partitioner = DFSPartitioner() - communities = await partitioner.partition(storage, max_units_per_community=5) + communities = list(partitioner.partition(storage, max_units_per_community=5)) assert communities == [] -@pytest.mark.asyncio -async def test_single_node(): +def test_single_node(): nodes = [("A", {"desc": "alone"})] edges = [] with tempfile.TemporaryDirectory() as tmpdir: @@ -29,21 +26,20 @@ async def test_single_node(): ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = DFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5 - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_small_graph(): +def test_small_graph(): """ 0 - 1 - 2 | | | @@ -68,17 +64,17 @@ async def test_small_graph(): ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = DFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4 - ) + )) - assert len(communities) <= 5 + assert len(communities) <= len(nodes) all_nodes = set() all_edges = set() for c in communities: diff --git a/tests/integration_tests/models/partitioner/test_ece_partitioner.py b/tests/integration_tests/models/partitioner/test_ece_partitioner.py index 93fc2ea1..2e35df97 100644 --- a/tests/integration_tests/models/partitioner/test_ece_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_ece_partitioner.py @@ -1,25 +1,22 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import ECEPartitioner, NetworkXStorage +from graphgen.models import ECEPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_ece_empty_graph(): +def test_ece_empty_graph(): """ECE partitioning on an empty graph should return an empty community list.""" with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="empty") partitioner = ECEPartitioner() - communities = await partitioner.partition( + communities = list(partitioner.partition( storage, max_units_per_community=5, unit_sampling="random" - ) + )) assert communities == [] -@pytest.mark.asyncio -async def test_ece_single_node(): +def test_ece_single_node(): """A single node must be placed in exactly one community under any edge-sampling strategy.""" nodes = [("A", {"desc": "alone", "length": 10, "loss": 0.1})] @@ -29,19 +26,18 @@ async def test_ece_single_node(): working_dir=tmpdir, namespace=f"single_{strategy}" ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5, unit_sampling=strategy - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_ece_small_graph_random(): +def test_ece_small_graph_random(): """ 2x3 grid graph: 0 — 1 — 2 @@ -63,14 +59,14 @@ async def test_ece_small_graph_random(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_random") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="random" - ) + )) # Basic integrity checks all_nodes = set() @@ -83,8 +79,7 @@ async def test_ece_small_graph_random(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_small_graph_min_loss(): +def test_ece_small_graph_min_loss(): """ Same grid graph, but using min_loss sampling. Edges with lower loss should be preferred during community expansion. @@ -105,14 +100,14 @@ async def test_ece_small_graph_min_loss(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_min") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="min_loss" - ) + )) all_nodes = set() all_edges = set() @@ -124,8 +119,7 @@ async def test_ece_small_graph_min_loss(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_small_graph_max_loss(): +def test_ece_small_graph_max_loss(): """ Same grid graph, but using max_loss sampling. Edges with higher loss should be preferred during community expansion. @@ -147,14 +141,14 @@ async def test_ece_small_graph_max_loss(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_max") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="max_loss" - ) + )) all_nodes = set() all_edges = set() @@ -166,8 +160,7 @@ async def test_ece_small_graph_max_loss(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_max_tokens_limit(): +def test_ece_max_tokens_limit(): """Ensure max_tokens_per_community is respected.""" # node id -> data node_data = {"A": {"length": 3000}, "B": {"length": 3000}, "C": {"length": 3000}} @@ -177,17 +170,17 @@ async def test_ece_max_tokens_limit(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="token_limit") for nid, ndata in node_data.items(): - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=10, max_tokens_per_community=5000, # 1 node (3000) + 1 edge (2000) = 5000 unit_sampling="random", - ) + )) # With a 5000-token budget we need at least two communities assert len(communities) >= 2 diff --git a/tests/test_event_profiles.py b/tests/test_event_profiles.py new file mode 100644 index 00000000..fd88539b --- /dev/null +++ b/tests/test_event_profiles.py @@ -0,0 +1,20 @@ +from graphgen.models.generator.support import build_generation_view, validate_support + + +def test_generation_view_hides_event_ids_and_maps_support_back_to_canonical_id(): + event_id = "event:opaque-hash" + item = { + "nodes": [ + (event_id, {"entity_type": "EVENT", "graph_role": "event", "description": "Migration completed：CASE-A7 event fact"}), + ("Operator", {"entity_type": "ACTOR", "description": "performed the action"}), + ], + "edges": [ + (event_id, "Operator", {"description": "participated"}), + ], + } + view = build_generation_view(item) + assert view["nodes"][0][0] == "Migration completed" + assert event_id not in str(view["nodes"]) + mapped = view["resolve_support"]({"cited": ["Migration completed"]}) + assert mapped == {"cited": [event_id]} + assert validate_support(mapped, set(view["canonical_nodes"])) == mapped diff --git a/tests/test_package_profiles.py b/tests/test_package_profiles.py new file mode 100644 index 00000000..da10c7e9 --- /dev/null +++ b/tests/test_package_profiles.py @@ -0,0 +1,30 @@ +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + + +def test_event_profile_assets_are_in_source_manifest_and_wheel(tmp_path): + import subprocess + import tarfile + import zipfile + + output = tmp_path / "dist" + subprocess.run(["uv", "build", str(REPO), "--out-dir", str(output)], check=True, capture_output=True, text=True) + wheel = next(output.glob("*.whl")) + sdist = next(output.glob("*.tar.gz")) + expected = { + "pms_event.v1/profile.json", + "pms_event.v1/prompt.zh.txt", + "generic_event.v1/profile.json", + "generic_event.v1/prompt.zh.txt", + "example_event.v1/profile.json", + "example_event.v1/prompt.zh.txt", + "pms_policy.v2/atomic.json", + "pms_policy.v3/multi_hop.json", + } + with zipfile.ZipFile(wheel) as archive: + wheel_files = archive.namelist() + with tarfile.open(sdist) as archive: + sdist_files = archive.getnames() + assert all(any(path.endswith(item) for path in wheel_files) for item in expected) + assert all(any(path.endswith(item) for path in sdist_files) for item in expected)