feat: 钉钉群聊飞书文档收集与知识图谱系统初始提交

- 通过悟空(dws CLI)拉取dc战略问题研究院+创新组两个群的消息(377条) - 提取190个飞书链接、18个文件附件 - 下载HTML/MD/XLSX等报告文件到output/downloaded-files/ - 构建知识图谱(JSON+HTML可视化) - 生成Obsidian知识库(28个页面,7大主题) - 生成花园世界全量汇总报告 - 所有脚本路径改为相对路径,便于迁移
2026-06-02 20:24:25 +08:00
commit fe5505343e
77 changed files with 20155 additions and 0 deletions
@@ -0,0 +1,282 @@
+import sys, io, json, re
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+
+import os
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
+
+with open(os.path.join(PROJECT_ROOT, "data", "raw-messages", "all_messages_combined.json"), "r", encoding="utf-8") as f:
+    data = json.load(f)
+
+feishu_re = re.compile(r"https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+")
+file_re = re.compile(r"\[文件\]\s+(.+?)\s+fileId:\s+(\S+)")
+kimi_re = re.compile(r"https?://[a-zA-Z0-9.-]+\.ok\.kimi\.link/\S*")
+weixin_re = re.compile(r"https?://mp\.weixin\.qq\.com/\S+")
+github_re = re.compile(r"https?://github\.com/\S+")
+
+# Person registry
+persons = {}
+# Link registry
+all_links = []
+# File registry
+all_files = []
+# Knowledge entities
+entities = []
+# Relationships
+relationships = []
+
+def get_or_create_person(name, dingtalk_id=None):
+    if name not in persons:
+        persons[name] = {"name": name, "dingtalk_id": dingtalk_id, "groups": [], "contributions": 0, "topics": set()}
+    persons[name]["contributions"] += 1
+    if dingtalk_id:
+        persons[name]["dingtalk_id"] = dingtalk_id
+    return persons[name]
+
+for group_name, msgs in data.items():
+    group_label = "dc战略问题研究院" if group_name == "dc" else "创新组"
+    
+    for msg in msgs:
+        content = msg.get("content", "")
+        sender = msg.get("sender", "unknown")
+        time = msg.get("createTime", "")
+        sender_id = msg.get("senderOpenDingTalkId", "")
+        
+        person = get_or_create_person(sender, sender_id)
+        if group_label not in person["groups"]:
+            person["groups"].append(group_label)
+        
+        # Extract feishu links
+        for link in feishu_re.findall(content):
+            entry = {
+                "url": link, "sender": sender, "time": time, "group": group_label,
+                "type": "wiki" if "/wiki/" in link else "docx",
+                "domain": link.split("/")[2]
+            }
+            # Try to get context from surrounding text
+            ctx = content.replace(link, "").strip()[:100]
+            if ctx:
+                entry["context"] = ctx
+            all_links.append(entry)
+        
+        # Extract file attachments
+        fm = file_re.search(content)
+        if fm:
+            all_files.append({
+                "name": fm.group(1), "fileId": fm.group(2),
+                "sender": sender, "time": time, "group": group_label
+            })
+        
+        # Extract Kimi links
+        for link in kimi_re.findall(content):
+            desc = content.split("https")[0].strip()[:100]
+            all_links.append({
+                "url": link, "sender": sender, "time": time, "group": group_label,
+                "type": "kimi", "domain": "kimi.link", "context": desc
+            })
+        
+        # Extract WeChat articles
+        for link in weixin_re.findall(content):
+            desc = content.split("https")[0].strip()[:100]
+            all_links.append({
+                "url": link, "sender": sender, "time": time, "group": group_label,
+                "type": "weixin_article", "domain": "mp.weixin.qq.com", "context": desc
+            })
+
+# Deduplicate links by URL
+unique_links = {}
+for link in all_links:
+    url = link["url"]
+    if url not in unique_links:
+        unique_links[url] = {**link, "shared_count": 1}
+    else:
+        unique_links[url]["shared_count"] += 1
+
+# Build knowledge graph
+graph = {
+    "metadata": {
+        "generated": "2026-06-02",
+        "groups": ["dc战略问题研究院", "创新组"],
+        "time_range": "2026-05-01 ~ 2026-06-02",
+        "total_messages": len(data["dc"]) + len(data["cx"])
+    },
+    "persons": {},
+    "documents": [],
+    "files": [],
+    "topics": [],
+    "relationships": []
+}
+
+# Process persons
+for name, p in persons.items():
+    graph["persons"][name] = {
+        "name": name,
+        "groups": p["groups"],
+        "contributions": p["contributions"],
+        "dingtalk_id": p.get("dingtalk_id", "")
+    }
+
+# Process documents (feishu links)
+domain_map = {
+    "dianchukeji.feishu.cn": "dianchukeji (公司飞书)",
+    "fcnlycv6dd0w.feishu.cn": "fcnlycv6dd0w (研究组飞书)",
+    "ocnmca6f1o0p.feishu.cn": "ocnmca6f1o0p (另一飞书空间)",
+    "my.feishu.cn": "my.feishu.cn (个人飞书)"
+}
+
+for url, link in unique_links.items():
+    doc = {
+        "url": url,
+        "type": link["type"],
+        "domain": domain_map.get(link["domain"], link["domain"]),
+        "shared_by": link["sender"],
+        "first_shared": link["time"],
+        "group": link["group"],
+        "context": link.get("context", ""),
+        "share_count": link["shared_count"]
+    }
+    graph["documents"].append(doc)
+
+# Process files
+seen_files = set()
+for f in all_files:
+    key = f["fileId"]
+    if key not in seen_files:
+        seen_files.add(key)
+        graph["files"].append(f)
+
+# Build topic clusters
+# Group documents by sender and domain to identify topic clusters
+sender_domains = {}
+for doc in graph["documents"]:
+    key = (doc["shared_by"], doc["domain"])
+    if key not in sender_domains:
+        sender_domains[key] = []
+    sender_domains[key].append(doc)
+
+# Identify main topics based on content analysis
+topics = [
+    {
+        "name": "《我的花园世界》深度研究",
+        "description": "围绕《我的花园世界》的全方位研究，包括产品机制、商业化、资源经济、竞品对比",
+        "key_people": ["李志健", "黄静雯", "陈楚真", "傅明游"],
+        "documents_count": 0,
+        "files": ["主报告-我的花园世界.html", "2026-05-25_report_my_garden_world_us_project.html",
+                  "花园项目对策划的启示.md", "进一步的思考.md", "花园世界产品策略_GOS补充信息.md",
+                  "花园世界等相关产品25.10.28.xlsx"],
+        "key_concepts": ["低压经营", "极致消耗", "非战斗GVG", "社交拉氪", "大通服", "腰部底座放大"]
+    },
+    {
+        "name": "云湖工作室项目选择",
+        "description": "广州云湖工作室第一个转型项目的方向选择分析",
+        "key_people": ["黄静雯"],
+        "documents_count": 0,
+        "files": ["2026-05-yunhu-project-selection-strategy-report.html"],
+        "key_concepts": ["2个月MVP", "AI美术适配", "素材CPI验证", "欧美化花园经营"]
+    },
+    {
+        "name": "Game Analyst Agent 工具",
+        "description": "通用产品分析自动化工具，自动探索游戏界面、提取机制与数值",
+        "key_people": ["黄静雯"],
+        "documents_count": 0,
+        "files": ["game-analyst-agent-manual.html", "game-analyst-agent.zip"],
+        "key_concepts": ["自动界面探索", "pHash去重", "9级决策链", "多游戏对比", "决策报告生成"]
+    },
+    {
+        "name": "战略思维与方法论",
+        "description": "李志健关于幸存者偏差、筹码管理、生长逻辑的战略反思",
+        "key_people": ["李志健"],
+        "documents_count": 0,
+        "files": ["取败之道.md", "模型报告.html"],
+        "key_concepts": ["幸存者偏差", "筹码管理", "正期望博弈", "生长逻辑", "非对称博弈"]
+    },
+    {
+        "name": "AI落地与组织变革",
+        "description": "AI在游戏研发中的实际落地、效率提升与组织变革讨论",
+        "key_people": ["李志健", "林峰", "汪雄军", "上官成", "安东"],
+        "documents_count": 0,
+        "files": [],
+        "key_concepts": ["AI军备竞赛", "端到端提效", "组织变革", "AI工作小组"]
+    },
+    {
+        "name": "创新组日报与研发进展",
+        "description": "创新组成员的每日工作日报，涵盖二合类游戏研发、AI工具使用",
+        "key_people": ["王雨默", "韦译", "刘鹏", "卓泽", "徐锐"],
+        "documents_count": 0,
+        "files": ["工作流总结-王雨默.html"],
+        "key_concepts": ["二合类游戏", "AI关卡生成", "UI编辑器", "数值平衡", "竞品分析"]
+    },
+    {
+        "name": "战略组日报与知识产出",
+        "description": "战略研究组成员每日提交的研究日报",
+        "key_people": ["黄静雯", "陈楚真", "张家振", "夏莲", "莫润麟", "胡辉俊", "汪季"],
+        "documents_count": 0,
+        "files": [],
+        "key_concepts": ["竞品拆解", "知识卡片", "MECH/REPORT/ORG卡片", "Agent架构"]
+    }
+]
+
+graph["topics"] = topics
+
+# Build relationships
+rels = []
+
+# Person -> Topic relationships
+for topic in topics:
+    for person in topic["key_people"]:
+        rels.append({
+            "source": person, "target": topic["name"],
+            "type": "contributes_to", "weight": 1
+        })
+
+# Person -> Person relationships (co-contribution in same group)
+for group_name, msgs in data.items():
+    group_label = "dc战略问题研究院" if group_name == "dc" else "创新组"
+    msg_senders = set(m.get("sender", "") for m in msgs)
+    for p1 in msg_senders:
+        for p2 in msg_senders:
+            if p1 < p2:
+                rels.append({
+                    "source": p1, "target": p2,
+                    "type": "co_group_member",
+                    "group": group_label, "weight": 1
+                })
+
+graph["relationships"] = rels
+
+# Save graph
+with open(os.path.join(PROJECT_ROOT, "output", "knowledge-graph", "knowledge_graph.json"), "w", encoding="utf-8") as f:
+    json.dump(graph, f, ensure_ascii=False, indent=2, default=str)
+
+# Print summary
+print("=== Knowledge Graph Summary ===")
+print("Persons: %d" % len(graph["persons"]))
+print("Documents (feishu links): %d" % len(graph["documents"]))
+print("Files: %d" % len(graph["files"]))
+print("Topics: %d" % len(graph["topics"]))
+print("Relationships: %d" % len(graph["relationships"]))
+
+print("\n=== Persons ===")
+for name, p in sorted(graph["persons"].items(), key=lambda x: -x[1]["contributions"]):
+    print("  %s: %d msgs, groups=%s" % (name, p["contributions"], ", ".join(p["groups"])))
+
+print("\n=== Documents by Domain ===")
+domain_counts = {}
+for doc in graph["documents"]:
+    d = doc["domain"]
+    domain_counts[d] = domain_counts.get(d, 0) + 1
+for d, c in sorted(domain_counts.items(), key=lambda x: -x[1]):
+    print("  %s: %d links" % (d, c))
+
+print("\n=== Files ===")
+for f in graph["files"]:
+    print("  %s (by %s, %s)" % (f["name"], f["sender"], f["time"]))
+
+print("\n=== Topics ===")
+for t in graph["topics"]:
+    print("  %s: %s" % (t["name"], t["description"][:60]))
+    print("    Key people: %s" % ", ".join(t["key_people"]))
+    print("    Key concepts: %s" % ", ".join(t["key_concepts"][:4]))
+
+print("\nSaved to knowledge_graph.json")
+