Evilom
93 lines
5.1 KiB
Python
93 lines
5.1 KiB
Python
"""Step 2: 从消息中提取飞书链接和文件附件"""
|
|
import argparse, json, os, re, sys
|
|
from datetime import datetime
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from paths import RAW_DIR, LINKS_DIR, normalize_messages
|
|
|
|
FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
|
|
FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
|
|
KIMI_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.ok\.kimi\.link/\S*')
|
|
MEDIA_RE = re.compile(r'mediaId=([^\s)]+)')
|
|
|
|
def extract(messages):
|
|
feishu, files, kimi, media = [], [], [], []
|
|
seen_f, seen_files, seen_media = set(), set(), set()
|
|
for grp, msgs in messages.items():
|
|
for m in msgs:
|
|
c, s, t = m.get("content",""), m.get("sender","?"), m.get("createTime","")
|
|
for url in FEISHU_RE.findall(c):
|
|
if url not in seen_f:
|
|
seen_f.add(url)
|
|
dm = re.search(r'feishu\.cn/(?:wiki|docx)/([A-Za-z0-9]+)', url)
|
|
feishu.append({"url":url,"doc_id":dm.group(1) if dm else "","type":"wiki" if "/wiki/" in url else "docx","sender":s,"time":t,"group":grp})
|
|
for name, fid in FILE_RE.findall(c):
|
|
if fid not in seen_files:
|
|
seen_files.add(fid)
|
|
files.append({"name":name.strip(),"fileId":fid,"sender":s,"time":t,"group":grp})
|
|
for url in KIMI_RE.findall(c):
|
|
kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
|
|
for resource_id in MEDIA_RE.findall(c):
|
|
message_id = m.get("openMessageId", "")
|
|
key = message_id or resource_id
|
|
if key not in seen_media:
|
|
seen_media.add(key)
|
|
media.append({
|
|
"resourceId": resource_id,
|
|
"messageId": message_id,
|
|
"conversationId": m.get("openConversationId", ""),
|
|
"mediaType": "video" if "[视频消息]" in c else "image",
|
|
"sender": s, "time": t, "group": grp,
|
|
})
|
|
return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi,"chat_media":media}
|
|
|
|
def load_json(path, default):
|
|
if not os.path.isfile(path): return default
|
|
with open(path,"r",encoding="utf-8") as f: return json.load(f)
|
|
|
|
def merge_by_key(old_items, new_items, key):
|
|
merged = list(old_items)
|
|
seen = set()
|
|
for item in old_items:
|
|
value = item.get(key, "") if isinstance(item, dict) else ""
|
|
if value and value not in seen:
|
|
seen.add(value)
|
|
for item in new_items:
|
|
value = item.get(key, "") if isinstance(item, dict) else ""
|
|
if value and value not in seen:
|
|
merged.append(item); seen.add(value)
|
|
return merged
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
|
|
path = os.path.join(RAW_DIR, "all_messages_combined.json")
|
|
if not os.path.isfile(path): print(f"ERROR: {path} not found"); sys.exit(1)
|
|
with open(path,"r",encoding="utf-8") as f: messages = normalize_messages(json.load(f))
|
|
print(f"消息: {sum(len(v) for v in messages.values())} 条")
|
|
r = extract(messages)
|
|
if args.incremental:
|
|
old_links = load_json(os.path.join(LINKS_DIR,"all_feishu_links.json"), {}).get("links",[])
|
|
old_files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json"), [])
|
|
old_kimi = load_json(os.path.join(LINKS_DIR,"all_kimi_links.json"), [])
|
|
old_media = load_json(os.path.join(LINKS_DIR,"all_chat_media.json"), [])
|
|
r["feishu_links"] = merge_by_key(old_links, r["feishu_links"], "doc_id")
|
|
r["file_attachments"] = merge_by_key(old_files, r["file_attachments"], "fileId")
|
|
r["kimi_links"] = merge_by_key(old_kimi, r["kimi_links"], "url")
|
|
r["chat_media"] = merge_by_key(old_media, r["chat_media"], "messageId")
|
|
print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, 媒体: {len(r['chat_media'])}, Kimi: {len(r['kimi_links'])}")
|
|
for grp in messages:
|
|
gl = [l for l in r['feishu_links'] if l['group']==grp]
|
|
gf = [f for f in r['file_attachments'] if f['group']==grp]
|
|
print(f" [{grp}] 飞书:{len(gl)} 附件:{len(gf)}")
|
|
os.makedirs(LINKS_DIR, exist_ok=True)
|
|
with open(os.path.join(LINKS_DIR,"all_feishu_links.json"),"w",encoding="utf-8") as f:
|
|
json.dump({"date":datetime.now().strftime("%Y-%m-%d"),"total":len(r['feishu_links']),"links":r['feishu_links']},f,ensure_ascii=False,indent=2)
|
|
with open(os.path.join(LINKS_DIR,"all_file_attachments.json"),"w",encoding="utf-8") as f:
|
|
json.dump(r['file_attachments'],f,ensure_ascii=False,indent=2)
|
|
with open(os.path.join(LINKS_DIR,"all_kimi_links.json"),"w",encoding="utf-8") as f:
|
|
json.dump(r['kimi_links'],f,ensure_ascii=False,indent=2)
|
|
with open(os.path.join(LINKS_DIR,"all_chat_media.json"),"w",encoding="utf-8") as f:
|
|
json.dump(r['chat_media'],f,ensure_ascii=False,indent=2)
|
|
print(f"保存至: {LINKS_DIR}")
|
|
|
|
if __name__ == "__main__": main()
|