chore: collect latest dingTalk feishu docs

skills/dingtalk-feishu-collector/scripts/paths.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+import json
 
 def find_project_root():
     """向上查找项目根目录（包含 data/ 目录的最顶层）"""
@@ -56,12 +57,59 @@ def find_dws():
 def find_lark_cli():
     """查找 lark-cli"""
     import shutil
+    root = find_project_root()
+    ext = ".exe" if os.name == "nt" else ""
+
+    local = os.path.join(root, "tools", "lark-cli", "lark-cli" + ext)
+    if os.path.isfile(local):
+        return local
+
     found = shutil.which("lark-cli")
     if found:
         return found
     print("ERROR: 找不到 lark-cli，请运行 npm install -g @larksuite/cli", file=sys.stderr)
     sys.exit(1)
 
+def load_groups():
+    """读取群配置，优先使用 data/group_config.json，失败时返回默认群。"""
+    default = {
+        "dc战略问题研究院": "cidoUneRB4Db8TAXaTrKxkQAw==",
+        "创新组": "cidMuM+itt5PeY7xNSWsv3M0g==",
+        "天龙计划X小游戏矩阵": "cid/Vb3hSjdv4sIIs8Fp7sC4g==",
+    }
+    path = os.path.join(find_project_root(), "data", "group_config.json")
+    if not os.path.isfile(path):
+        return default
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            raw = json.load(f).get("groups", {})
+        groups = {}
+        for name, value in raw.items():
+            if isinstance(value, dict):
+                cid = value.get("cid") or value.get("id")
+            else:
+                cid = value
+            if cid:
+                groups[name] = cid
+        return groups or default
+    except Exception as e:
+        print(f"WARN: 读取群配置失败，使用默认配置: {e}", file=sys.stderr)
+        return default
+
+def normalize_messages(messages):
+    """兼容 list 和按群 dict 两种历史消息文件格式。"""
+    if isinstance(messages, dict):
+        return messages
+    grouped = {}
+    if not isinstance(messages, list):
+        return grouped
+    for msg in messages:
+        if not isinstance(msg, dict):
+            continue
+        group = msg.get("group") or msg.get("groupName") or msg.get("openConversationId") or "未分组"
+        grouped.setdefault(group, []).append(msg)
+    return grouped
+
 # 常用路径
 PROJECT_ROOT = find_project_root()
 DATA_DIR = os.path.join(PROJECT_ROOT, "data")

skills/dingtalk-feishu-collector/scripts/run_all.py

@@ -28,33 +28,36 @@ def main():
     p.add_argument("--skip-download",action="store_true"); p.add_argument("--skip-kb",action="store_true")
     args = p.parse_args()
     t0 = time.time()
+    failures = 0
 
     s1 = []
     if args.full: s1.append("--full")
     elif args.since: s1 += ["--since", args.since]
     elif args.days: s1 += ["--days", str(args.days)]
     elif args.quick: s1 += ["--days", "1"]
-    run("step1_collect_messages.py", s1)
+    failures += 1 if run("step1_collect_messages.py", s1) else 0
 
     s2 = [] if args.full else ["--incremental"]
-    run("step2_extract_links.py", s2)
+    failures += 1 if run("step2_extract_links.py", s2) else 0
 
     s3 = [] if args.full else ["--incremental"]
-    run("step3_fetch_feishu_docs.py", s3)
+    failures += 1 if run("step3_fetch_feishu_docs.py", s3) else 0
 
     if not args.skip_download:
         s4 = [] if args.full else ["--incremental"]
-        run("step4_download_files.py", s4)
+        failures += 1 if run("step4_download_files.py", s4) else 0
 
     s5 = []
     if args.quick: s5 += ["--since", "today", "--brief"]
     elif args.since: s5 += ["--since", args.since]
     elif args.days: s5 += ["--days", str(args.days)]
-    run("step5_generate_summary.py", s5)
+    failures += 1 if run("step5_generate_summary.py", s5) else 0
 
     if not args.skip_kb:
-        run("step6_update_knowledge_base.py")
+        failures += 1 if run("step6_update_knowledge_base.py") else 0
 
-    print(f"\n{'='*50}\n  全部完成! ({time.time()-t0:.1f}s)\n{'='*50}")
+    status = "全部完成" if failures == 0 else f"完成，但有 {failures} 步失败"
+    print(f"\n{'='*50}\n  {status}! ({time.time()-t0:.1f}s)\n{'='*50}")
+    return 1 if failures else 0
 
-if __name__ == "__main__": main()
+if __name__ == "__main__": sys.exit(main())

skills/dingtalk-feishu-collector/scripts/step1_collect_messages.py

@@ -15,7 +15,7 @@ import sys
 from datetime import datetime, timedelta
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from paths import DATA_DIR, RAW_DIR, find_dws
+from paths import RAW_DIR, find_dws, load_groups, normalize_messages
 
 DWS = find_dws()
 
@@ -51,10 +51,7 @@ def collect_group(group_name, group_id, since):
         print(f"  总计: {len(result)} 条 ({times[0]} ~ {times[-1]})")
     return result
 
-GROUPS = {
-    "dc战略问题研究院": "cidoUneRB4Db8TAXaTrKxkQAw==",
-    "创新组": "cidMuM+itt5PeY7xNSWsv3M0g==",
-}
+GROUPS = load_groups()
 
 def main():
     parser = argparse.ArgumentParser()
@@ -79,7 +76,7 @@ def main():
     existing = {}
     if os.path.isfile(combined_path):
         with open(combined_path, "r", encoding="utf-8") as f:
-            existing = json.load(f)
+            existing = normalize_messages(json.load(f))
 
     new_count = 0
     for name, gid in GROUPS.items():

skills/dingtalk-feishu-collector/scripts/step2_extract_links.py

@@ -2,7 +2,7 @@
 import argparse, json, os, re, sys
 from datetime import datetime
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from paths import RAW_DIR, LINKS_DIR
+from paths import RAW_DIR, LINKS_DIR, normalize_messages
 
 FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
 FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
@@ -27,13 +27,37 @@ def extract(messages):
                 kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
     return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi}
 
+def load_json(path, default):
+    if not os.path.isfile(path): return default
+    with open(path,"r",encoding="utf-8") as f: return json.load(f)
+
+def merge_by_key(old_items, new_items, key):
+    merged = list(old_items)
+    seen = set()
+    for item in old_items:
+        value = item.get(key, "") if isinstance(item, dict) else ""
+        if value and value not in seen:
+            seen.add(value)
+    for item in new_items:
+        value = item.get(key, "") if isinstance(item, dict) else ""
+        if value and value not in seen:
+            merged.append(item); seen.add(value)
+    return merged
+
 def main():
     p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
     path = os.path.join(RAW_DIR, "all_messages_combined.json")
     if not os.path.isfile(path): print(f"ERROR: {path} not found"); sys.exit(1)
-    with open(path,"r",encoding="utf-8") as f: messages = json.load(f)
+    with open(path,"r",encoding="utf-8") as f: messages = normalize_messages(json.load(f))
     print(f"消息: {sum(len(v) for v in messages.values())} 条")
     r = extract(messages)
+    if args.incremental:
+        old_links = load_json(os.path.join(LINKS_DIR,"all_feishu_links.json"), {}).get("links",[])
+        old_files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json"), [])
+        old_kimi = load_json(os.path.join(LINKS_DIR,"all_kimi_links.json"), [])
+        r["feishu_links"] = merge_by_key(old_links, r["feishu_links"], "doc_id")
+        r["file_attachments"] = merge_by_key(old_files, r["file_attachments"], "fileId")
+        r["kimi_links"] = merge_by_key(old_kimi, r["kimi_links"], "url")
     print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, Kimi: {len(r['kimi_links'])}")
     for grp in messages:
         gl = [l for l in r['feishu_links'] if l['group']==grp]

skills/dingtalk-feishu-collector/scripts/step3_fetch_feishu_docs.py

@@ -4,12 +4,29 @@ from datetime import datetime
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from paths import LINKS_DIR, DOCS_DIR, find_lark_cli
 
-def resolve_wiki(node_id, lark):
+def lark_api(lark, path, params=None, timeout=30):
+    cmd = [lark, "api", "GET", path, "--format", "json"]
+    if params:
+        cmd += ["--params", json.dumps(params, ensure_ascii=False)]
+    r = subprocess.run(cmd, capture_output=True, timeout=timeout)
+    out = r.stdout.decode("utf-8", errors="replace")
+    err = r.stderr.decode("utf-8", errors="replace")
+    if r.returncode != 0:
+        return None, err or out
     try:
-        r = subprocess.run([lark,"api","GET",f"/open-apis/wiki/v2/spaces/get_node?token={node_id}","--format","json"],capture_output=True,text=True,timeout=15)
-        if r.returncode==0: return json.loads(r.stdout).get("data",{}).get("node",{}).get("obj_token","")
-    except: pass
-    return ""
+        data = json.loads(out)
+    except Exception as e:
+        return None, f"JSON解析失败: {e}"
+    if data.get("ok") is False or data.get("code", 0) not in (0, None):
+        return None, data.get("error") or data
+    return data, None
+
+def resolve_wiki(node_id, lark):
+    data, _ = lark_api(lark, "/open-apis/wiki/v2/spaces/get_node", params={"token": node_id}, timeout=15)
+    if not data:
+        return "", ""
+    node = data.get("data",{}).get("node",{})
+    return node.get("obj_token",""), node.get("title","")
 
 def extract_text(obj):
     if not obj: return ""
@@ -31,14 +48,36 @@ def blocks_to_md(blocks, doc_id):
     return "\n\n".join(lines)
 
 def fetch_doc(doc_id, doc_type, lark):
-    real_id = resolve_wiki(doc_id, lark) if doc_type=="wiki" else doc_id
-    if not real_id: return f"# {doc_id}\n\n> wiki解析失败"
-    try:
-        r = subprocess.run([lark,"api","GET",f"/open-apis/docx/v1/documents/{real_id}/blocks","--format","json"],capture_output=True,text=True,timeout=30)
-        if r.returncode!=0: return f"# {doc_id}\n\n> 获取失败"
-        blocks = json.loads(r.stdout).get("data",{}).get("items",[])
-        return blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
-    except Exception as e: return f"# {doc_id}\n\n> {e}"
+    real_id, wiki_title = (doc_id, "")
+    if doc_type=="wiki":
+        real_id, wiki_title = resolve_wiki(doc_id, lark)
+    if not real_id:
+        return f"# {doc_id}\n\n> wiki解析失败"
+    data, error = lark_api(lark, f"/open-apis/docx/v1/documents/{real_id}/blocks", timeout=30)
+    if error:
+        return f"# {doc_id}\n\n> 获取失败: {error}"
+    blocks = data.get("data",{}).get("items",[]) if data else []
+    md = blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
+    if wiki_title:
+        md = md.replace(f"# {doc_id}", f"# {wiki_title}", 1)
+    return md
+
+def should_fetch(doc_id, existing, docs_dir):
+    old = existing.get(doc_id)
+    if not old:
+        return True
+    if old.get("content_length", 0) <= 200:
+        return True
+    path = os.path.join(docs_dir, f"{doc_id}.md")
+    if not os.path.isfile(path):
+        return True
+    with open(path, "r", encoding="utf-8") as f:
+        head = f.read(500)
+    return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档"])
+
+def is_error_markdown(md):
+    head = md[:1000]
+    return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档", "forBidden", "permission denied"])
 
 def main():
     p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
@@ -59,14 +98,19 @@ def main():
     new_count = 0
     for link in all_links:
         did = link.get("doc_id","")
-        if not did or did in existing: continue
+        if not did or not should_fetch(did, existing, DOCS_DIR): continue
         print(f"  {did} ({link.get('sender','?')})")
         md = fetch_doc(did, link.get("type","docx"), lark)
+        fetched = not is_error_markdown(md)
+        old = existing.get(did)
+        if old and old.get("content_length", 0) > 200 and not fetched:
+            print("    WARN: 获取失败，保留已有正文")
+            continue
         with open(os.path.join(DOCS_DIR,f"{did}.md"),"w",encoding="utf-8") as f: f.write(md)
         title = ""
         for line in md.split("\n"):
             if line.startswith("# "): title = line[2:].strip(); break
-        existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":True}
+        existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":fetched}
         new_count += 1
 
     with open(content_path,"w",encoding="utf-8") as f:

skills/dingtalk-feishu-collector/scripts/step4_download_files.py

@@ -16,10 +16,14 @@ def main():
     for fi in files:
         fid, name = fi.get("fileId",""), fi.get("name","unknown")
         if not fid: continue
+        safe_name = os.path.basename(name)
         ext = os.path.splitext(name)[1].lower()
         subdir = "html-md" if ext in HTML_MD else "other"
         out = os.path.join(DOWNLOAD_DIR, subdir)
         os.makedirs(out, exist_ok=True)
+        if args.incremental and os.path.isfile(os.path.join(out, safe_name)):
+            print(f"  SKIP {safe_name}")
+            continue
         print(f"  {name} -> {subdir}/")
         try:
             r = subprocess.run([dws,"drive","download","--node",fid,"--output",out],capture_output=True,timeout=120)

skills/dingtalk-feishu-collector/scripts/step5_generate_summary.py

@@ -3,7 +3,7 @@ import argparse, json, os, sys
 from datetime import datetime, timedelta
 from collections import defaultdict
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from paths import DATA_DIR, LINKS_DIR, DOCS_DIR, REPORTS_DIR
+from paths import DATA_DIR, LINKS_DIR, DOCS_DIR, REPORTS_DIR, normalize_messages
 
 def load_json(path):
     if not os.path.isfile(path): return None
@@ -15,7 +15,7 @@ def main():
     p.add_argument("--full",action="store_true"); p.add_argument("--brief",action="store_true")
     args = p.parse_args()
 
-    messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
+    messages = normalize_messages(load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {})
     content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
     files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json")) or []
     docs = {}

skills/dingtalk-feishu-collector/scripts/step6_update_knowledge_base.py

@@ -3,7 +3,7 @@ import json, os, sys
 from datetime import datetime
 from collections import defaultdict
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from paths import LINKS_DIR, DATA_DIR, DOCS_DIR, OBSIDIAN_DIR, KG_DIR
+from paths import LINKS_DIR, DATA_DIR, DOCS_DIR, OBSIDIAN_DIR, KG_DIR, normalize_messages
 
 def safe_name(n):
     for c in '<>:"/\\|?*': n=n.replace(c,'_')
@@ -19,7 +19,7 @@ def load_json(p):
 
 def main():
     content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
-    messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
+    messages = normalize_messages(load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {})
     docs = content.get("documents",[])
 
     print(f"文档: {len(docs)}, 消息: {sum(len(v) for v in messages.values())}")