Evilom/dc-docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: collect complete Feishu media assets

Browse Source
This commit is contained in:
Evilom
2026-07-11 12:30:54 +08:00
parent 3ea50649f6
commit bbafe770b3
662 changed files with 15404 additions and 14017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/dingtalk-feishu-collector/scripts/fetch_feishu_full.py
+21 -7
View File
@@ -4,15 +4,22 @@ import argparse, json, os, subprocess, sys, time
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR
from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR, find_lark_cli
LARK = r'C:\Users\admin\AppData\Roaming\npm\node_modules\@larksuite\cli\bin\lark-cli.exe'
LARK = find_lark_cli()
IMAGES_DIR = os.path.join(OUTPUT_DIR, 'feishu-images')
def lark_api(path, timeout=30):
r = subprocess.run([LARK, 'api', 'GET', path, '--format', 'json'], capture_output=True, timeout=timeout)
def lark_api(path, params=None, timeout=30):
cmd = [LARK, 'api', 'GET', path, '--format', 'json']
if params:
cmd += ['--params', json.dumps(params, ensure_ascii=False)]
r = subprocess.run(cmd, capture_output=True, timeout=timeout)
out = r.stdout.decode('utf-8', errors='replace')
try: return json.loads(out)
try:
result = json.loads(out)
if result.get('ok') is True:
return {'code': 0, 'data': result.get('data', {})}
return result
except: return None
def download_media(token, local_path, timeout=30):
@@ -29,7 +36,7 @@ def download_media(token, local_path, timeout=30):
except: return False
def resolve_wiki(node_id):
data = lark_api(f'/open-apis/wiki/v2/spaces/get_node?token={node_id}')
data = lark_api('/open-apis/wiki/v2/spaces/get_node', params={'token': node_id})
if data and data.get('code') == 0:
node = data.get('data', {}).get('node', {})
return node.get('obj_token', ''), node.get('title', '')
@@ -192,7 +199,14 @@ def main():
if not args.rebuild and doc_id in existing:
old = existing[doc_id]
if old.get('fetched', False) and old.get('content_length', 0) > 200 and old.get('images_downloaded', 0) > 0:
image_blocks = old.get('image_blocks')
images_downloaded = old.get('images_downloaded')
media_complete = (
image_blocks is not None
and images_downloaded is not None
and images_downloaded >= image_blocks
)
if old.get('fetched', False) and old.get('content_length', 0) > 200 and media_complete:
skip += 1
continue
skills/dingtalk-feishu-collector/scripts/run_all.py
+1 -1
View File
@@ -41,7 +41,7 @@ def main():
failures += 1 if run("step2_extract_links.py", s2) else 0
s3 = [] if args.full else ["--incremental"]
failures += 1 if run("step3_fetch_feishu_docs.py", s3) else 0
failures += 1 if run("fetch_feishu_full.py") else 0
if not args.skip_download:
s4 = [] if args.full else ["--incremental"]
skills/dingtalk-feishu-collector/scripts/step2_extract_links.py
+21 -4
View File
@@ -7,10 +7,11 @@ from paths import RAW_DIR, LINKS_DIR, normalize_messages
FEISHU_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+')
FILE_RE = re.compile(r'\[文件\]\s+(.+?)\s+fileId:\s+(\S+)')
KIMI_RE = re.compile(r'https?://[a-zA-Z0-9.-]+\.ok\.kimi\.link/\S*')
MEDIA_RE = re.compile(r'mediaId=([^\s)]+)')
def extract(messages):
feishu, files, kimi = [], [], []
seen_f, seen_files = set(), set()
feishu, files, kimi, media = [], [], [], []
seen_f, seen_files, seen_media = set(), set(), set()
for grp, msgs in messages.items():
for m in msgs:
c, s, t = m.get("content",""), m.get("sender","?"), m.get("createTime","")
@@ -25,7 +26,19 @@ def extract(messages):
files.append({"name":name.strip(),"fileId":fid,"sender":s,"time":t,"group":grp})
for url in KIMI_RE.findall(c):
kimi.append({"url":url,"desc":c.split("https")[0].strip()[:80],"sender":s,"group":grp})
return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi}
for resource_id in MEDIA_RE.findall(c):
message_id = m.get("openMessageId", "")
key = message_id or resource_id
if key not in seen_media:
seen_media.add(key)
media.append({
"resourceId": resource_id,
"messageId": message_id,
"conversationId": m.get("openConversationId", ""),
"mediaType": "video" if "[视频消息]" in c else "image",
"sender": s, "time": t, "group": grp,
})
return {"feishu_links":feishu,"file_attachments":files,"kimi_links":kimi,"chat_media":media}
def load_json(path, default):
if not os.path.isfile(path): return default
@@ -55,10 +68,12 @@ def main():
old_links = load_json(os.path.join(LINKS_DIR,"all_feishu_links.json"), {}).get("links",[])
old_files = load_json(os.path.join(LINKS_DIR,"all_file_attachments.json"), [])
old_kimi = load_json(os.path.join(LINKS_DIR,"all_kimi_links.json"), [])
old_media = load_json(os.path.join(LINKS_DIR,"all_chat_media.json"), [])
r["feishu_links"] = merge_by_key(old_links, r["feishu_links"], "doc_id")
r["file_attachments"] = merge_by_key(old_files, r["file_attachments"], "fileId")
r["kimi_links"] = merge_by_key(old_kimi, r["kimi_links"], "url")
print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, Kimi: {len(r['kimi_links'])}")
r["chat_media"] = merge_by_key(old_media, r["chat_media"], "messageId")
print(f"飞书: {len(r['feishu_links'])}, 附件: {len(r['file_attachments'])}, 媒体: {len(r['chat_media'])}, Kimi: {len(r['kimi_links'])}")
for grp in messages:
gl = [l for l in r['feishu_links'] if l['group']==grp]
gf = [f for f in r['file_attachments'] if f['group']==grp]
@@ -70,6 +85,8 @@ def main():
json.dump(r['file_attachments'],f,ensure_ascii=False,indent=2)
with open(os.path.join(LINKS_DIR,"all_kimi_links.json"),"w",encoding="utf-8") as f:
json.dump(r['kimi_links'],f,ensure_ascii=False,indent=2)
with open(os.path.join(LINKS_DIR,"all_chat_media.json"),"w",encoding="utf-8") as f:
json.dump(r['chat_media'],f,ensure_ascii=False,indent=2)
print(f"保存至: {LINKS_DIR}")
if __name__ == "__main__": main()
skills/dingtalk-feishu-collector/scripts/step4_download_files.py
+33 -1
View File
@@ -30,6 +30,38 @@ def main():
if r.returncode==0: new+=1
else: print(f" FAIL")
except Exception as e: print(f" ERR: {e}")
print(f"\n下载 {new} 个 -> {DOWNLOAD_DIR}")
media_path = os.path.join(LINKS_DIR, "all_chat_media.json")
media = []
if os.path.isfile(media_path):
with open(media_path, "r", encoding="utf-8") as f: media = json.load(f)
media_dir = os.path.join(DOWNLOAD_DIR, "chat-media")
os.makedirs(media_dir, exist_ok=True)
media_new = 0
for item in media:
resource_id = item.get("resourceId", "")
message_id = item.get("messageId", "")
conversation_id = item.get("conversationId", "")
if not resource_id or not message_id or not conversation_id:
print(f" MEDIA FAIL 缺少下载标识: {message_id or resource_id}")
continue
ext = ".mp4" if item.get("mediaType") == "video" else ".jpg"
stamp = item.get("time", "unknown").replace(":", "").replace(" ", "_").replace("-", "")
safe_id = "".join(ch for ch in message_id if ch.isalnum())[-12:] or "media"
output = os.path.join(media_dir, f"{stamp}_{safe_id}{ext}")
if args.incremental and os.path.isfile(output) and os.path.getsize(output) > 0:
print(f" SKIP {os.path.basename(output)}")
continue
cmd = [dws, "chat", "message", "download-media", "--type", "mediaId",
"--resource-id", resource_id, "--message-id", message_id,
"--open-conversation-id", conversation_id, "--output", output, "-y"]
try:
r = subprocess.run(cmd, capture_output=True, timeout=120)
if r.returncode == 0 and os.path.isfile(output) and os.path.getsize(output) > 0:
media_new += 1
else:
print(f" MEDIA FAIL {message_id}")
except Exception as e: print(f" MEDIA ERR {message_id}: {e}")
print(f"\n下载附件 {new} 个、聊天媒体 {media_new} 个 -> {DOWNLOAD_DIR}")
if __name__ == "__main__": main()