chore: collect latest dingTalk feishu docs

skills/dingtalk-feishu-collector/scripts/step2_extract_links.py

@@ -2,7 +2,7 @@
 import argparse, json, os, re, sys
 from datetime import datetime
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from paths import RAW_DIR, LINKS_DIR
+from paths import RAW_DIR, LINKS_DIR, normalize_messages
 
 FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
 FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
@@ -27,13 +27,37 @@ def extract(messages):
                 kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
     return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi}
 
+def load_json(path, default):
+    if not os.path.isfile(path): return default
+    with open(path,"r",encoding="utf-8") as f: return json.load(f)
+
+def merge_by_key(old_items, new_items, key):
+    merged = list(old_items)
+    seen = set()
+    for item in old_items:
+        value = item.get(key, "") if isinstance(item, dict) else ""
+        if value and value not in seen:
+            seen.add(value)
+    for item in new_items:
+        value = item.get(key, "") if isinstance(item, dict) else ""
+        if value and value not in seen:
+            merged.append(item); seen.add(value)
+    return merged
+
 def main():
     p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
     path = os.path.join(RAW_DIR, "all_messages_combined.json")
     if not os.path.isfile(path): print(f"ERROR: {path} not found"); sys.exit(1)
-    with open(path,"r",encoding="utf-8") as f: messages = json.load(f)
+    with open(path,"r",encoding="utf-8") as f: messages = normalize_messages(json.load(f))
     print(f"消息: {sum(len(v) for v in messages.values())} 条")
     r = extract(messages)
+    if args.incremental:
+        old_links = load_json(os.path.join(LINKS_DIR,"all_feishu_links.json"), {}).get("links",[])
+        old_files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json"), [])
+        old_kimi = load_json(os.path.join(LINKS_DIR,"all_kimi_links.json"), [])
+        r["feishu_links"] = merge_by_key(old_links, r["feishu_links"], "doc_id")
+        r["file_attachments"] = merge_by_key(old_files, r["file_attachments"], "fileId")
+        r["kimi_links"] = merge_by_key(old_kimi, r["kimi_links"], "url")
     print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, Kimi: {len(r['kimi_links'])}")
     for grp in messages:
         gl = [l for l in r['feishu_links'] if l['group']==grp]