feat: add dingtalk-feishu-collector SOP skill
Reusable 6-step pipeline for collecting DingTalk group messages, extracting Feishu doc links, fetching content, and generating summaries. Skills structure: - SKILL.md: trigger rules, workflow, config reference - config.yaml: group IDs, collection settings - scripts/paths.py: shared path resolution - scripts/step1-6: modular pipeline steps - scripts/run_all.py: one-click runner
This commit is contained in:
Evilom
@@ -0,0 +1,75 @@
|
||||
"""共享路径和配置工具"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
def find_project_root():
|
||||
"""向上查找项目根目录（包含 data/ 目录的最顶层）"""
|
||||
# 从当前脚本位置开始
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# 向上查找，找到包含 data/ 目录的目录
|
||||
d = script_dir
|
||||
for _ in range(5): # 最多向上5级
|
||||
if os.path.isdir(os.path.join(d, "data")):
|
||||
return d
|
||||
parent = os.path.dirname(d)
|
||||
if parent == d:
|
||||
break
|
||||
d = parent
|
||||
|
||||
# 如果找不到，假设项目根在 scripts/ 的上一级
|
||||
return os.path.dirname(os.path.dirname(script_dir))
|
||||
|
||||
def find_dws():
|
||||
"""查找 dws CLI"""
|
||||
import shutil
|
||||
|
||||
# 1. 环境变量
|
||||
env = os.environ.get("DWS_PATH")
|
||||
if env and os.path.isfile(env):
|
||||
return env
|
||||
|
||||
# 2. 项目 tools/ 目录
|
||||
root = find_project_root()
|
||||
local = os.path.join(root, "tools", "dws.exe")
|
||||
if os.path.isfile(local):
|
||||
return local
|
||||
|
||||
# 3. 悟空内置
|
||||
candidates = [
|
||||
os.path.expanduser(r"~\.real\.bin\dws\bin\dws.exe"),
|
||||
r"C:\Program Files\Wukong\0.9.51-26052503\bin\dws.exe",
|
||||
]
|
||||
for p in candidates:
|
||||
if os.path.isfile(p):
|
||||
return p
|
||||
|
||||
# 4. PATH
|
||||
found = shutil.which("dws")
|
||||
if found:
|
||||
return found
|
||||
|
||||
print("ERROR: 找不到 dws CLI，请设置 DWS_PATH 环境变量或安装悟空", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
def find_lark_cli():
|
||||
"""查找 lark-cli"""
|
||||
import shutil
|
||||
found = shutil.which("lark-cli")
|
||||
if found:
|
||||
return found
|
||||
print("ERROR: 找不到 lark-cli，请运行 npm install -g @larksuite/cli", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# 常用路径
|
||||
PROJECT_ROOT = find_project_root()
|
||||
DATA_DIR = os.path.join(PROJECT_ROOT, "data")
|
||||
RAW_DIR = os.path.join(DATA_DIR, "raw-messages")
|
||||
LINKS_DIR = os.path.join(DATA_DIR, "links")
|
||||
OUTPUT_DIR = os.path.join(PROJECT_ROOT, "output")
|
||||
DOCS_DIR = os.path.join(OUTPUT_DIR, "feishu-docs")
|
||||
REPORTS_DIR = os.path.join(OUTPUT_DIR, "reports")
|
||||
DOWNLOAD_DIR = os.path.join(OUTPUT_DIR, "downloaded-files")
|
||||
OBSIDIAN_DIR = os.path.join(OUTPUT_DIR, "obsidian-vault")
|
||||
KG_DIR = os.path.join(OUTPUT_DIR, "knowledge-graph")
|
||||
@@ -0,0 +1,60 @@
|
||||
"""一键执行完整采集流程
|
||||
|
||||
用法:
|
||||
python run_all.py # 增量（最近3天）
|
||||
python run_all.py --days 1 # 最近1天
|
||||
python run_all.py --full # 全量
|
||||
python run_all.py --quick # 只看今天
|
||||
python run_all.py --skip-download # 跳过文件下载
|
||||
python run_all.py --skip-kb # 跳过知识库更新
|
||||
"""
|
||||
|
||||
import argparse, subprocess, sys, os, time
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
def run(script, args=None):
|
||||
cmd = [sys.executable, os.path.join(SCRIPT_DIR, script)] + (args or [])
|
||||
print(f"\n{'='*50}\n {script} {' '.join(args or [])}\n{'='*50}")
|
||||
t = time.time()
|
||||
r = subprocess.run(cmd)
|
||||
print(f" {'OK' if r.returncode==0 else 'WARN'} ({time.time()-t:.1f}s)")
|
||||
return r.returncode
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--full",action="store_true"); p.add_argument("--days",type=int)
|
||||
p.add_argument("--since",type=str); p.add_argument("--quick",action="store_true")
|
||||
p.add_argument("--skip-download",action="store_true"); p.add_argument("--skip-kb",action="store_true")
|
||||
args = p.parse_args()
|
||||
t0 = time.time()
|
||||
|
||||
s1 = []
|
||||
if args.full: s1.append("--full")
|
||||
elif args.since: s1 += ["--since", args.since]
|
||||
elif args.days: s1 += ["--days", str(args.days)]
|
||||
elif args.quick: s1 += ["--days", "1"]
|
||||
run("step1_collect_messages.py", s1)
|
||||
|
||||
s2 = [] if args.full else ["--incremental"]
|
||||
run("step2_extract_links.py", s2)
|
||||
|
||||
s3 = [] if args.full else ["--incremental"]
|
||||
run("step3_fetch_feishu_docs.py", s3)
|
||||
|
||||
if not args.skip_download:
|
||||
s4 = [] if args.full else ["--incremental"]
|
||||
run("step4_download_files.py", s4)
|
||||
|
||||
s5 = []
|
||||
if args.quick: s5 += ["--since", "today", "--brief"]
|
||||
elif args.since: s5 += ["--since", args.since]
|
||||
elif args.days: s5 += ["--days", str(args.days)]
|
||||
run("step5_generate_summary.py", s5)
|
||||
|
||||
if not args.skip_kb:
|
||||
run("step6_update_knowledge_base.py")
|
||||
|
||||
print(f"\n{'='*50}\n 全部完成! ({time.time()-t0:.1f}s)\n{'='*50}")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Step 1: 从钉钉群拉取消息
|
||||
|
||||
用法:
|
||||
python step1_collect_messages.py # 增量（最近3天）
|
||||
python step1_collect_messages.py --days 1 # 增量（最近1天）
|
||||
python step1_collect_messages.py --full # 全量拉取
|
||||
python step1_collect_messages.py --since "2026-06-05 00:00:00"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import DATA_DIR, RAW_DIR, find_dws
|
||||
|
||||
DWS = find_dws()
|
||||
|
||||
def fetch_messages(group_id, time_str, forward="true", limit=200):
|
||||
cmd = [DWS, "chat", "message", "list", "--group", group_id,
|
||||
"--time", time_str, "--forward", forward, "--limit", str(limit), "--format", "json"]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, timeout=60)
|
||||
output = result.stdout.decode("utf-8", errors="replace")
|
||||
json_start = output.find("{")
|
||||
if json_start < 0: return []
|
||||
data = json.loads(output[json_start:])
|
||||
return data.get("result", {}).get("messages", [])
|
||||
except Exception as e:
|
||||
print(f" WARN: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
def collect_group(group_name, group_id, since):
|
||||
print(f"\n=== {group_name} (since {since}) ===")
|
||||
all_msgs = {}
|
||||
msgs = fetch_messages(group_id, since, "true")
|
||||
for m in msgs: all_msgs[m["openMessageId"]] = m
|
||||
print(f" 正向: {len(msgs)} 条")
|
||||
if len(msgs) >= 180:
|
||||
times = sorted([m["createTime"] for m in msgs])
|
||||
if times:
|
||||
msgs2 = fetch_messages(group_id, times[len(times)//2], "false")
|
||||
for m in msgs2: all_msgs[m["openMessageId"]] = m
|
||||
print(f" 反向补充: {len(msgs2)} 条")
|
||||
result = list(all_msgs.values())
|
||||
if result:
|
||||
times = sorted([m["createTime"] for m in result])
|
||||
print(f" 总计: {len(result)} 条 ({times[0]} ~ {times[-1]})")
|
||||
return result
|
||||
|
||||
GROUPS = {
|
||||
"dc战略问题研究院": "cidoUneRB4Db8TAXaTrKxkQAw==",
|
||||
"创新组": "cidMuM+itt5PeY7xNSWsv3M0g==",
|
||||
}
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--days", type=int)
|
||||
parser.add_argument("--since", type=str)
|
||||
parser.add_argument("--full", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.since:
|
||||
since = args.since
|
||||
elif args.full:
|
||||
since = "2026-05-01 00:00:00"
|
||||
elif args.days:
|
||||
since = (datetime.now() - timedelta(days=args.days)).strftime("%Y-%m-%d 00:00:00")
|
||||
else:
|
||||
since = (datetime.now() - timedelta(days=3)).strftime("%Y-%m-%d 00:00:00")
|
||||
|
||||
print(f"dws: {DWS}\n时间: {since} ~ now")
|
||||
|
||||
os.makedirs(RAW_DIR, exist_ok=True)
|
||||
combined_path = os.path.join(RAW_DIR, "all_messages_combined.json")
|
||||
existing = {}
|
||||
if os.path.isfile(combined_path):
|
||||
with open(combined_path, "r", encoding="utf-8") as f:
|
||||
existing = json.load(f)
|
||||
|
||||
new_count = 0
|
||||
for name, gid in GROUPS.items():
|
||||
new_msgs = collect_group(name, gid, since)
|
||||
if name not in existing: existing[name] = []
|
||||
existing_ids = {m["openMessageId"] for m in existing[name]}
|
||||
added = sum(1 for m in new_msgs if m["openMessageId"] not in existing_ids)
|
||||
for m in new_msgs:
|
||||
if m["openMessageId"] not in existing_ids:
|
||||
existing[name].append(m)
|
||||
existing_ids.add(m["openMessageId"])
|
||||
print(f" 新增: {added} 条 (总计: {len(existing[name])})")
|
||||
new_count += added
|
||||
|
||||
with open(combined_path, "w", encoding="utf-8") as f:
|
||||
json.dump(existing, f, ensure_ascii=False, indent=2)
|
||||
total = sum(len(v) for v in existing.values())
|
||||
print(f"\n完成! 新增 {new_count} 条，总计 {total} 条 -> {combined_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Step 2: 从消息中提取飞书链接和文件附件"""
|
||||
import argparse, json, os, re, sys
|
||||
from datetime import datetime
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import RAW_DIR, LINKS_DIR
|
||||
|
||||
FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
|
||||
FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
|
||||
KIMI_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.ok\.kimi\.link/\S*')
|
||||
|
||||
def extract(messages):
|
||||
feishu, files, kimi = [], [], []
|
||||
seen_f, seen_files = set(), set()
|
||||
for grp, msgs in messages.items():
|
||||
for m in msgs:
|
||||
c, s, t = m.get("content",""), m.get("sender","?"), m.get("createTime","")
|
||||
for url in FEISHU_RE.findall(c):
|
||||
if url not in seen_f:
|
||||
seen_f.add(url)
|
||||
dm = re.search(r'feishu\.cn/(?:wiki|docx)/([A-Za-z0-9]+)', url)
|
||||
feishu.append({"url":url,"doc_id":dm.group(1) if dm else "","type":"wiki" if "/wiki/" in url else "docx","sender":s,"time":t,"group":grp})
|
||||
for name, fid in FILE_RE.findall(c):
|
||||
if fid not in seen_files:
|
||||
seen_files.add(fid)
|
||||
files.append({"name":name.strip(),"fileId":fid,"sender":s,"time":t,"group":grp})
|
||||
for url in KIMI_RE.findall(c):
|
||||
kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
|
||||
return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi}
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
|
||||
path = os.path.join(RAW_DIR, "all_messages_combined.json")
|
||||
if not os.path.isfile(path): print(f"ERROR: {path} not found"); sys.exit(1)
|
||||
with open(path,"r",encoding="utf-8") as f: messages = json.load(f)
|
||||
print(f"消息: {sum(len(v) for v in messages.values())} 条")
|
||||
r = extract(messages)
|
||||
print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, Kimi: {len(r['kimi_links'])}")
|
||||
for grp in messages:
|
||||
gl = [l for l in r['feishu_links'] if l['group']==grp]
|
||||
gf = [f for f in r['file_attachments'] if f['group']==grp]
|
||||
print(f" [{grp}] 飞书:{len(gl)} 附件:{len(gf)}")
|
||||
os.makedirs(LINKS_DIR, exist_ok=True)
|
||||
with open(os.path.join(LINKS_DIR,"all_feishu_links.json"),"w",encoding="utf-8") as f:
|
||||
json.dump({"date":datetime.now().strftime("%Y-%m-%d"),"total":len(r['feishu_links']),"links":r['feishu_links']},f,ensure_ascii=False,indent=2)
|
||||
with open(os.path.join(LINKS_DIR,"all_file_attachments.json"),"w",encoding="utf-8") as f:
|
||||
json.dump(r['file_attachments'],f,ensure_ascii=False,indent=2)
|
||||
with open(os.path.join(LINKS_DIR,"all_kimi_links.json"),"w",encoding="utf-8") as f:
|
||||
json.dump(r['kimi_links'],f,ensure_ascii=False,indent=2)
|
||||
print(f"保存至: {LINKS_DIR}")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Step 3: 拉取飞书文档内容"""
|
||||
import argparse, json, os, subprocess, sys, re
|
||||
from datetime import datetime
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import LINKS_DIR, DOCS_DIR, find_lark_cli
|
||||
|
||||
def resolve_wiki(node_id, lark):
|
||||
try:
|
||||
r = subprocess.run([lark,"api","GET",f"/open-apis/wiki/v2/spaces/get_node?token={node_id}","--format","json"],capture_output=True,text=True,timeout=15)
|
||||
if r.returncode==0: return json.loads(r.stdout).get("data",{}).get("node",{}).get("obj_token","")
|
||||
except: pass
|
||||
return ""
|
||||
|
||||
def extract_text(obj):
|
||||
if not obj: return ""
|
||||
return "".join(e.get("text_run",{}).get("content","") for e in obj.get("elements",[])).strip()
|
||||
|
||||
def blocks_to_md(blocks, doc_id):
|
||||
lines = [f"# {doc_id}\n", f"Source: https://feishu.cn/docx/{doc_id}\n"]
|
||||
for b in blocks:
|
||||
bt = b.get("block_type",0)
|
||||
if bt==2: t=extract_text(b.get("text",{})); [lines.append(t)] if t else None
|
||||
elif bt in range(3,10): t=extract_text(b.get(f"heading{bt-2}",{}) or b.get("text",{})); lines.append(f"{'#'*(bt-2)} {t}") if t else None
|
||||
elif bt==10: t=extract_text(b.get("bullet",{}) or b.get("text",{})); lines.append(f"- {t}") if t else None
|
||||
elif bt==11: t=extract_text(b.get("ordered",{}) or b.get("text",{})); lines.append(f"1. {t}") if t else None
|
||||
elif bt==14: t=extract_text(b.get("quote",{}) or b.get("text",{})); lines.append(f"> {t}") if t else None
|
||||
elif bt==15: t=extract_text(b.get("code",{}) or b.get("text",{})); lines.append(f"```\n{t}\n```") if t else None
|
||||
else:
|
||||
for k in ["text","heading1","heading2","heading3","bullet","ordered","quote","code"]:
|
||||
if k in b: t=extract_text(b[k]); [lines.append(t)] if t else None; break
|
||||
return "\n\n".join(lines)
|
||||
|
||||
def fetch_doc(doc_id, doc_type, lark):
|
||||
real_id = resolve_wiki(doc_id, lark) if doc_type=="wiki" else doc_id
|
||||
if not real_id: return f"# {doc_id}\n\n> wiki解析失败"
|
||||
try:
|
||||
r = subprocess.run([lark,"api","GET",f"/open-apis/docx/v1/documents/{real_id}/blocks","--format","json"],capture_output=True,text=True,timeout=30)
|
||||
if r.returncode!=0: return f"# {doc_id}\n\n> 获取失败"
|
||||
blocks = json.loads(r.stdout).get("data",{}).get("items",[])
|
||||
return blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
|
||||
except Exception as e: return f"# {doc_id}\n\n> {e}"
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
|
||||
links_path = os.path.join(LINKS_DIR,"all_feishu_links.json")
|
||||
if not os.path.isfile(links_path): print("ERROR: 先运行 step2"); sys.exit(1)
|
||||
with open(links_path,"r",encoding="utf-8") as f: all_links = json.load(f).get("links",[])
|
||||
print(f"链接: {len(all_links)} 个")
|
||||
|
||||
content_path = os.path.join(LINKS_DIR,"all_feishu_content.json")
|
||||
existing = {}
|
||||
if os.path.isfile(content_path):
|
||||
with open(content_path,"r",encoding="utf-8") as f:
|
||||
for d in json.load(f).get("documents",[]): existing[d.get("doc_id","")] = d
|
||||
print(f"已拉取: {len(existing)}")
|
||||
|
||||
lark = find_lark_cli()
|
||||
os.makedirs(DOCS_DIR, exist_ok=True)
|
||||
new_count = 0
|
||||
for link in all_links:
|
||||
did = link.get("doc_id","")
|
||||
if not did or did in existing: continue
|
||||
print(f" {did} ({link.get('sender','?')})")
|
||||
md = fetch_doc(did, link.get("type","docx"), lark)
|
||||
with open(os.path.join(DOCS_DIR,f"{did}.md"),"w",encoding="utf-8") as f: f.write(md)
|
||||
title = ""
|
||||
for line in md.split("\n"):
|
||||
if line.startswith("# "): title = line[2:].strip(); break
|
||||
existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":True}
|
||||
new_count += 1
|
||||
|
||||
with open(content_path,"w",encoding="utf-8") as f:
|
||||
json.dump({"date":datetime.now().strftime("%Y-%m-%d"),"total":len(existing),"documents":list(existing.values())},f,ensure_ascii=False,indent=2)
|
||||
print(f"\n新增 {new_count}，总计 {len(existing)} -> {DOCS_DIR}")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Step 4: 下载钉钉群文件附件"""
|
||||
import argparse, json, os, subprocess, sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import LINKS_DIR, DOWNLOAD_DIR, find_dws
|
||||
|
||||
HTML_MD = {".html",".htm",".md",".markdown",".txt"}
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
|
||||
path = os.path.join(LINKS_DIR,"all_file_attachments.json")
|
||||
if not os.path.isfile(path): print("ERROR: 先运行 step2"); sys.exit(1)
|
||||
with open(path,"r",encoding="utf-8") as f: files = json.load(f)
|
||||
print(f"附件: {len(files)} 个")
|
||||
dws = find_dws()
|
||||
new = 0
|
||||
for fi in files:
|
||||
fid, name = fi.get("fileId",""), fi.get("name","unknown")
|
||||
if not fid: continue
|
||||
ext = os.path.splitext(name)[1].lower()
|
||||
subdir = "html-md" if ext in HTML_MD else "other"
|
||||
out = os.path.join(DOWNLOAD_DIR, subdir)
|
||||
os.makedirs(out, exist_ok=True)
|
||||
print(f" {name} -> {subdir}/")
|
||||
try:
|
||||
r = subprocess.run([dws,"drive","download","--node",fid,"--output",out],capture_output=True,timeout=120)
|
||||
if r.returncode==0: new+=1
|
||||
else: print(f" FAIL")
|
||||
except Exception as e: print(f" ERR: {e}")
|
||||
print(f"\n下载 {new} 个 -> {DOWNLOAD_DIR}")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Step 5: 生成结构化总结"""
|
||||
import argparse, json, os, sys
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import DATA_DIR, LINKS_DIR, DOCS_DIR, REPORTS_DIR
|
||||
|
||||
def load_json(path):
|
||||
if not os.path.isfile(path): return None
|
||||
with open(path,"r",encoding="utf-8") as f: return json.load(f)
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--since",type=str); p.add_argument("--days",type=int)
|
||||
p.add_argument("--full",action="store_true"); p.add_argument("--brief",action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
|
||||
content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
|
||||
files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json")) or []
|
||||
docs = {}
|
||||
for d in content.get("documents",[]):
|
||||
did = d.get("doc_id","") or d.get("url","")
|
||||
if did: docs[did] = d
|
||||
|
||||
if args.since:
|
||||
since = datetime.now().strftime("%Y-%m-%d 00:00:00") if args.since=="today" else (datetime.now()-timedelta(days=1)).strftime("%Y-%m-%d 00:00:00") if args.since=="yesterday" else args.since
|
||||
messages = {g:[m for m in msgs if m.get("createTime","")>=since] for g,msgs in messages.items()}
|
||||
elif args.days:
|
||||
since = (datetime.now()-timedelta(days=args.days)).strftime("%Y-%m-%d 00:00:00")
|
||||
messages = {g:[m for m in msgs if m.get("createTime","")>=since] for g,msgs in messages.items()}
|
||||
|
||||
lines = [f"# 钉钉群聊飞书文档采集报告", f"> 生成: {datetime.now().strftime('%Y-%m-%d %H:%M')}", ""]
|
||||
total_m = sum(len(v) for v in messages.values())
|
||||
lines += ["## 数据总览","","| 指标 | 数量 |","|------|------|",
|
||||
f"| 消息 | {total_m} |", f"| 文档 | {len(docs)} |", f"| 附件 | {len(files)} |", ""]
|
||||
|
||||
for grp, msgs in messages.items():
|
||||
if not msgs: continue
|
||||
times = sorted([m.get("createTime","") for m in msgs])
|
||||
gd = [d for d in docs.values() if d.get("group")==grp]
|
||||
gf = [f for f in files if f.get("group")==grp]
|
||||
lines += [f"### {grp}", f"- 消息: {len(msgs)} ({times[0]} ~ {times[-1]})", f"- 文档: {len(gd)}", f"- 附件: {len(gf)}", ""]
|
||||
if args.brief: continue
|
||||
senders = defaultdict(lambda: {"m":0,"d":0})
|
||||
for m in msgs: senders[m.get("sender","?")]["m"]+=1
|
||||
for d in gd: senders[d.get("sender","?")]["d"]+=1
|
||||
lines += ["| 发送者 | 消息 | 文档 |","|--------|------|------|"]
|
||||
for s,st in sorted(senders.items(),key=lambda x:-(x[1]["m"]+x[1]["d"])):
|
||||
lines.append(f"| {s} | {st['m']} | {st['d']} |")
|
||||
lines.append("")
|
||||
lines.append("**最新消息:**")
|
||||
for m in sorted(msgs,key=lambda x:x.get("createTime",""),reverse=True)[:10]:
|
||||
c = m.get('content','')[:80].replace('\n',' ')
|
||||
lines.append(f"- [{m.get('createTime','')}] **{m.get('sender','?')}**: {c}")
|
||||
lines.append("")
|
||||
|
||||
os.makedirs(REPORTS_DIR, exist_ok=True)
|
||||
report = "\n".join(lines)
|
||||
for name in ["latest_summary.md", f"summary_{datetime.now().strftime('%Y%m%d')}.md"]:
|
||||
with open(os.path.join(REPORTS_DIR,name),"w",encoding="utf-8") as f: f.write(report)
|
||||
print(f"报告已生成: {REPORTS_DIR}")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Step 6: 更新知识库（Obsidian + 知识图谱）"""
|
||||
import json, os, sys
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import LINKS_DIR, DATA_DIR, DOCS_DIR, OBSIDIAN_DIR, KG_DIR
|
||||
|
||||
def safe_name(n):
|
||||
for c in '<>:"/\\|?*': n=n.replace(c,'_')
|
||||
return n[:80]
|
||||
|
||||
def w(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path,"w",encoding="utf-8") as f: f.write(content)
|
||||
|
||||
def load_json(p):
|
||||
if not os.path.isfile(p): return None
|
||||
with open(p,"r",encoding="utf-8") as f: return json.load(f)
|
||||
|
||||
def main():
|
||||
content = load_json(os.path.join(LINKS_DIR,"all_feishu_content.json")) or {}
|
||||
messages = load_json(os.path.join(DATA_DIR,"raw-messages","all_messages_combined.json")) or {}
|
||||
docs = content.get("documents",[])
|
||||
|
||||
print(f"文档: {len(docs)}, 消息: {sum(len(v) for v in messages.values())}")
|
||||
|
||||
# Obsidian
|
||||
by_group = defaultdict(list)
|
||||
for d in docs: by_group[d.get("group","未分组")].append(d)
|
||||
by_sender = defaultdict(list)
|
||||
for d in docs: by_sender[d.get("sender","?")].append(d)
|
||||
|
||||
moc = ["---","tags: [MOC, 飞书文档]",f"created: {datetime.now().strftime('%Y-%m-%d')}","---","","# 飞书文档知识库","",f"> 更新: {datetime.now().strftime('%Y-%m-%d %H:%M')}",""]
|
||||
for grp, gd in sorted(by_group.items()):
|
||||
moc.append(f"## {grp}")
|
||||
for d in sorted(gd,key=lambda x:x.get("time",""),reverse=True):
|
||||
t = d.get("title","") or d.get("doc_id","")
|
||||
moc.append(f"- [[{safe_name(t)}]] ({d.get('sender','?')}, {d.get('time','')})")
|
||||
moc.append("")
|
||||
w(os.path.join(OBSIDIAN_DIR,"00-MOC","飞书文档索引.md"),"\n".join(moc))
|
||||
|
||||
for sender, sd in sorted(by_sender.items()):
|
||||
lines = ["---",f"tags: [人物, {sender}]","---",f"# {sender}",f"\n贡献: {len(sd)} 篇\n"]
|
||||
for d in sorted(sd,key=lambda x:x.get("time",""),reverse=True):
|
||||
t = d.get("title","") or d.get("doc_id","")
|
||||
lines.append(f"- [[{safe_name(t)}]] ({d.get('time','')})")
|
||||
w(os.path.join(OBSIDIAN_DIR,"04-人物",f"{safe_name(sender)}.md"),"\n".join(lines))
|
||||
|
||||
for d in docs:
|
||||
did = d.get("doc_id","")
|
||||
fpath = os.path.join(DOCS_DIR,f"{did}.md")
|
||||
if not os.path.isfile(fpath): continue
|
||||
with open(fpath,"r",encoding="utf-8") as f: c = f.read()
|
||||
t = d.get("title","") or did
|
||||
fm = ["---",f"tags: [飞书文档, {d.get('group','')}]",f"sender: {d.get('sender','')}",f"date: {d.get('time','')}",f"source: {d.get('url','')}","---",""]
|
||||
w(os.path.join(OBSIDIAN_DIR,"01-产品研究",f"{safe_name(t)}.md"),"\n".join(fm)+c)
|
||||
print(f" Obsidian: {len(docs)} 文档, {len(by_sender)} 人物")
|
||||
|
||||
# 知识图谱
|
||||
nodes, edges, nids = [], [], set()
|
||||
for s in {d.get("sender","") for d in docs} | {m.get("sender","") for msgs in messages.values() for m in msgs}:
|
||||
if s: nid=f"person:{s}"; nodes.append({"id":nid,"type":"person","label":s}); nids.add(nid)
|
||||
for d in docs:
|
||||
did,t = d.get("doc_id",""), d.get("title","") or d.get("doc_id","")
|
||||
nid = f"doc:{did}"
|
||||
if nid not in nids: nodes.append({"id":nid,"type":"document","label":t[:50]}); nids.add(nid)
|
||||
s = d.get("sender","")
|
||||
if s: edges.append({"source":f"person:{s}","target":nid,"type":"authored"})
|
||||
g = d.get("group","")
|
||||
if g:
|
||||
gid=f"group:{g}"
|
||||
if gid not in nids: nodes.append({"id":gid,"type":"group","label":g}); nids.add(gid)
|
||||
edges.append({"source":gid,"target":nid,"type":"contains"})
|
||||
os.makedirs(KG_DIR, exist_ok=True)
|
||||
with open(os.path.join(KG_DIR,"knowledge_graph.json"),"w",encoding="utf-8") as f:
|
||||
json.dump({"date":datetime.now().strftime("%Y-%m-%d"),"nodes":nodes,"edges":edges},f,ensure_ascii=False,indent=2)
|
||||
print(f" 图谱: {len(nodes)} 节点, {len(edges)} 关系")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
Reference in New Issue
Block a user