Evilom/dc-docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: collect latest dingTalk feishu docs

Browse Source
This commit is contained in:
Evilom
2026-07-03 14:39:58 +08:00
parent d60ee0bc7c
commit 3ea50649f6
167 changed files with 23587 additions and 1759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/dingtalk-feishu-collector/config.yaml
+1
View File
@@ -4,6 +4,7 @@
groups:
dc战略问题研究院: "cidoUneRB4Db8TAXaTrKxkQAw=="
创新组: "cidMuM+itt5PeY7xNSWsv3M0g=="
天龙计划X小游戏矩阵: "cid/Vb3hSjdv4sIIs8Fp7sC4g=="
collection:
# 首次全量拉取的起始时间
skills/dingtalk-feishu-collector/scripts/paths.py
+48
View File
@@ -2,6 +2,7 @@
import os
import sys
import json
def find_project_root():
"""向上查找项目根目录（包含 data/ 目录的最顶层）"""
@@ -56,12 +57,59 @@ def find_dws():
def find_lark_cli():
"""查找 lark-cli"""
import shutil
root = find_project_root()
ext = ".exe" if os.name == "nt" else ""
local = os.path.join(root, "tools", "lark-cli", "lark-cli" + ext)
if os.path.isfile(local):
return local
found = shutil.which("lark-cli")
if found:
return found
print("ERROR: 找不到 lark-cli，请运行 npm install -g @larksuite/cli", file=sys.stderr)
sys.exit(1)
def load_groups():
"""读取群配置，优先使用 data/group_config.json，失败时返回默认群。"""
default = {
"dc战略问题研究院": "cidoUneRB4Db8TAXaTrKxkQAw==",
"创新组": "cidMuM+itt5PeY7xNSWsv3M0g==",
"天龙计划X小游戏矩阵": "cid/Vb3hSjdv4sIIs8Fp7sC4g==",
}
path = os.path.join(find_project_root(), "data", "group_config.json")
if not os.path.isfile(path):
return default
try:
with open(path, "r", encoding="utf-8") as f:
raw = json.load(f).get("groups", {})
groups = {}
for name, value in raw.items():
if isinstance(value, dict):
cid = value.get("cid") or value.get("id")
else:
cid = value
if cid:
groups[name] = cid
return groups or default
except Exception as e:
print(f"WARN: 读取群配置失败，使用默认配置: {e}", file=sys.stderr)
return default
def normalize_messages(messages):
"""兼容 list 和按群 dict 两种历史消息文件格式。"""
if isinstance(messages, dict):
return messages
grouped = {}
if not isinstance(messages, list):
return grouped
for msg in messages:
if not isinstance(msg, dict):
continue
group = msg.get("group") or msg.get("groupName") or msg.get("openConversationId") or "未分组"
grouped.setdefault(group, []).append(msg)
return grouped
# 常用路径
PROJECT_ROOT = find_project_root()
DATA_DIR = os.path.join(PROJECT_ROOT, "data")
skills/dingtalk-feishu-collector/scripts/run_all.py
+11 -8
View File
@@ -28,33 +28,36 @@ def main():
p.add_argument("--skip-download",action="store_true"); p.add_argument("--skip-kb",action="store_true")
args = p.parse_args()
t0 = time.time()
failures = 0
s1 = []
if args.full: s1.append("--full")
elif args.since: s1 += ["--since", args.since]
elif args.days: s1 += ["--days", str(args.days)]
elif args.quick: s1 += ["--days", "1"]
run("step1_collect_messages.py", s1)
failures += 1 if run("step1_collect_messages.py", s1) else 0
s2 = [] if args.full else ["--incremental"]
run("step2_extract_links.py", s2)
failures += 1 if run("step2_extract_links.py", s2) else 0
s3 = [] if args.full else ["--incremental"]
run("step3_fetch_feishu_docs.py", s3)
failures += 1 if run("step3_fetch_feishu_docs.py", s3) else 0
if not args.skip_download:
s4 = [] if args.full else ["--incremental"]
run("step4_download_files.py", s4)
failures += 1 if run("step4_download_files.py", s4) else 0
s5 = []
if args.quick: s5 += ["--since", "today", "--brief"]
elif args.since: s5 += ["--since", args.since]
elif args.days: s5 += ["--days", str(args.days)]
run("step5_generate_summary.py", s5)
failures += 1 if run("step5_generate_summary.py", s5) else 0
if not args.skip_kb:
run("step6_update_knowledge_base.py")
failures += 1 if run("step6_update_knowledge_base.py") else 0
print(f"\n{'='*50}\n 全部完成! ({time.time()-t0:.1f}s)\n{'='*50}")
status = "全部完成" if failures == 0 else f"完成，但有 {failures} 步失败"
print(f"\n{'='*50}\n {status}! ({time.time()-t0:.1f}s)\n{'='*50}")
return 1 if failures else 0
if __name__ == "__main__": main()
if __name__ == "__main__": sys.exit(main())
skills/dingtalk-feishu-collector/scripts/step1_collect_messages.py
+3 -6
View File
@@ -15,7 +15,7 @@ import sys
from datetime import datetime, timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import DATA_DIR, RAW_DIR, find_dws
from paths import RAW_DIR, find_dws, load_groups, normalize_messages
DWS = find_dws()
@@ -51,10 +51,7 @@ def collect_group(group_name, group_id, since):
print(f" 总计: {len(result)} 条 ({times[0]} ~ {times[-1]})")
return result
GROUPS = {
"dc战略问题研究院": "cidoUneRB4Db8TAXaTrKxkQAw==",
"创新组": "cidMuM+itt5PeY7xNSWsv3M0g==",
}
GROUPS = load_groups()
def main():
parser = argparse.ArgumentParser()
@@ -79,7 +76,7 @@ def main():
existing = {}
if os.path.isfile(combined_path):
with open(combined_path, "r", encoding="utf-8") as f:
existing = json.load(f)
existing = normalize_messages(json.load(f))
new_count = 0
for name, gid in GROUPS.items():
skills/dingtalk-feishu-collector/scripts/step2_extract_links.py
+26 -2
View File
@@ -2,7 +2,7 @@
import argparse, json, os, re, sys
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import RAW_DIR, LINKS_DIR
from paths import RAW_DIR, LINKS_DIR, normalize_messages
FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
@@ -27,13 +27,37 @@ def extract(messages):
kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi}
def load_json(path, default):
if not os.path.isfile(path): return default
with open(path,"r",encoding="utf-8") as f: return json.load(f)
def merge_by_key(old_items, new_items, key):
merged = list(old_items)
seen = set()
for item in old_items:
value = item.get(key, "") if isinstance(item, dict) else ""
if value and value not in seen:
seen.add(value)
for item in new_items:
value = item.get(key, "") if isinstance(item, dict) else ""
if value and value not in seen:
merged.append(item); seen.add(value)
return merged
def main():
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
path = os.path.join(RAW_DIR, "all_messages_combined.json")
if not os.path.isfile(path): print(f"ERROR: {path} not found"); sys.exit(1)
with open(path,"r",encoding="utf-8") as f: messages = json.load(f)
with open(path,"r",encoding="utf-8") as f: messages = normalize_messages(json.load(f))
print(f"消息: {sum(len(v) for v in messages.values())} 条")
r = extract(messages)
if args.incremental:
old_links = load_json(os.path.join(LINKS_DIR,"all_feishu_links.json"), {}).get("links",[])
old_files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json"), [])
old_kimi = load_json(os.path.join(LINKS_DIR,"all_kimi_links.json"), [])
r["feishu_links"] = merge_by_key(old_links, r["feishu_links"], "doc_id")
r["file_attachments"] = merge_by_key(old_files, r["file_attachments"], "fileId")
r["kimi_links"] = merge_by_key(old_kimi, r["kimi_links"], "url")
print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, Kimi: {len(r['kimi_links'])}")
for grp in messages:
gl = [l for l in r['feishu_links'] if l['group']==grp]
skills/dingtalk-feishu-collector/scripts/step3_fetch_feishu_docs.py
+59 -15
View File
@@ -4,12 +4,29 @@ from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import LINKS_DIR, DOCS_DIR, find_lark_cli
def resolve_wiki(node_id, lark):
def lark_api(lark, path, params=None, timeout=30):
cmd = [lark, "api", "GET", path, "--format", "json"]
if params:
cmd += ["--params", json.dumps(params, ensure_ascii=False)]
r = subprocess.run(cmd, capture_output=True, timeout=timeout)
out = r.stdout.decode("utf-8", errors="replace")
err = r.stderr.decode("utf-8", errors="replace")
if r.returncode != 0:
return None, err or out
try:
r = subprocess.run([lark,"api","GET",f"/open-apis/wiki/v2/spaces/get_node?token={node_id}","--format","json"],capture_output=True,text=True,timeout=15)
if r.returncode==0: return json.loads(r.stdout).get("data",{}).get("node",{}).get("obj_token","")
except: pass
return ""
data = json.loads(out)
except Exception as e:
return None, f"JSON解析失败: {e}"
if data.get("ok") is False or data.get("code", 0) not in (0, None):
return None, data.get("error") or data
return data, None
def resolve_wiki(node_id, lark):
data, _ = lark_api(lark, "/open-apis/wiki/v2/spaces/get_node", params={"token": node_id}, timeout=15)
if not data:
return "", ""
node = data.get("data",{}).get("node",{})
return node.get("obj_token",""), node.get("title","")
def extract_text(obj):
if not obj: return ""
@@ -31,14 +48,36 @@ def blocks_to_md(blocks, doc_id):
return "\n\n".join(lines)
def fetch_doc(doc_id, doc_type, lark):
real_id = resolve_wiki(doc_id, lark) if doc_type=="wiki" else doc_id
if not real_id: return f"# {doc_id}\n\n> wiki解析失败"
try:
r = subprocess.run([lark,"api","GET",f"/open-apis/docx/v1/documents/{real_id}/blocks","--format","json"],capture_output=True,text=True,timeout=30)
if r.returncode!=0: return f"# {doc_id}\n\n> 获取失败"
blocks = json.loads(r.stdout).get("data",{}).get("items",[])
return blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
except Exception as e: return f"# {doc_id}\n\n> {e}"
real_id, wiki_title = (doc_id, "")
if doc_type=="wiki":
real_id, wiki_title = resolve_wiki(doc_id, lark)
if not real_id:
return f"# {doc_id}\n\n> wiki解析失败"
data, error = lark_api(lark, f"/open-apis/docx/v1/documents/{real_id}/blocks", timeout=30)
if error:
return f"# {doc_id}\n\n> 获取失败: {error}"
blocks = data.get("data",{}).get("items",[]) if data else []
md = blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
if wiki_title:
md = md.replace(f"# {doc_id}", f"# {wiki_title}", 1)
return md
def should_fetch(doc_id, existing, docs_dir):
old = existing.get(doc_id)
if not old:
return True
if old.get("content_length", 0) <= 200:
return True
path = os.path.join(docs_dir, f"{doc_id}.md")
if not os.path.isfile(path):
return True
with open(path, "r", encoding="utf-8") as f:
head = f.read(500)
return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档"])
def is_error_markdown(md):
head = md[:1000]
return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档", "forBidden", "permission denied"])
def main():
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
@@ -59,14 +98,19 @@ def main():
new_count = 0
for link in all_links:
did = link.get("doc_id","")
if not did or did in existing: continue
if not did or not should_fetch(did, existing, DOCS_DIR): continue
print(f" {did} ({link.get('sender','?')})")
md = fetch_doc(did, link.get("type","docx"), lark)
fetched = not is_error_markdown(md)
old = existing.get(did)
if old and old.get("content_length", 0) > 200 and not fetched:
print(" WARN: 获取失败，保留已有正文")
continue
with open(os.path.join(DOCS_DIR,f"{did}.md"),"w",encoding="utf-8") as f: f.write(md)
title = ""
for line in md.split("\n"):
if line.startswith("# "): title = line[2:].strip(); break
existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":True}
existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":fetched}
new_count += 1
with open(content_path,"w",encoding="utf-8") as f:
skills/dingtalk-feishu-collector/scripts/step4_download_files.py
+4
View File
@@ -16,10 +16,14 @@ def main():
for fi in files:
fid, name = fi.get("fileId",""), fi.get("name","unknown")
if not fid: continue
safe_name = os.path.basename(name)
ext = os.path.splitext(name)[1].lower()
subdir = "html-md" if ext in HTML_MD else "other"
out = os.path.join(DOWNLOAD_DIR, subdir)
os.makedirs(out, exist_ok=True)
if args.incremental and os.path.isfile(os.path.join(out, safe_name)):
print(f" SKIP {safe_name}")
continue
print(f" {name} -> {subdir}/")
try:
r = subprocess.run([dws,"drive","download","--node",fid,"--output",out],capture_output=True,timeout=120)
skills/dingtalk-feishu-collector/scripts/step5_generate_summary.py
+2 -2
View File
@@ -3,7 +3,7 @@ import argparse, json, os, sys
from datetime import datetime, timedelta
from collections import defaultdict
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import DATA_DIR, LINKS_DIR, DOCS_DIR, REPORTS_DIR
from paths import DATA_DIR, LINKS_DIR, DOCS_DIR, REPORTS_DIR, normalize_messages
def load_json(path):
if not os.path.isfile(path): return None
@@ -15,7 +15,7 @@ def main():
p.add_argument("--full",action="store_true"); p.add_argument("--brief",action="store_true")
args = p.parse_args()
messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
messages = normalize_messages(load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {})
content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json")) or []
docs = {}
skills/dingtalk-feishu-collector/scripts/step6_update_knowledge_base.py
+2 -2
View File
@@ -3,7 +3,7 @@ import json, os, sys
from datetime import datetime
from collections import defaultdict
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import LINKS_DIR, DATA_DIR, DOCS_DIR, OBSIDIAN_DIR, KG_DIR
from paths import LINKS_DIR, DATA_DIR, DOCS_DIR, OBSIDIAN_DIR, KG_DIR, normalize_messages
def safe_name(n):
for c in '<>:"/\\|?*': n=n.replace(c,'_')
@@ -19,7 +19,7 @@ def load_json(p):
def main():
content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
messages = normalize_messages(load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {})
docs = content.get("documents",[])
print(f"文档: {len(docs)}, 消息: {sum(len(v) for v in messages.values())}")