Evilom
236 lines
9.6 KiB
Python
236 lines
9.6 KiB
Python
"""飞书文档完整拉取器 - 处理所有 block 类型 + 图片下载"""
|
|
|
|
import argparse, json, os, subprocess, sys, time
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR
|
|
|
|
LARK = r'C:\Users\admin\AppData\Roaming\npm\node_modules\@larksuite\cli\bin\lark-cli.exe'
|
|
IMAGES_DIR = os.path.join(OUTPUT_DIR, 'feishu-images')
|
|
|
|
def lark_api(path, timeout=30):
|
|
r = subprocess.run([LARK, 'api', 'GET', path, '--format', 'json'], capture_output=True, timeout=timeout)
|
|
out = r.stdout.decode('utf-8', errors='replace')
|
|
try: return json.loads(out)
|
|
except: return None
|
|
|
|
def download_media(token, local_path, timeout=30):
|
|
"""下载飞书媒体文件 - lark-cli 要求相对路径，需用 cwd"""
|
|
if os.path.isfile(local_path): return True
|
|
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
|
filename = os.path.basename(local_path)
|
|
try:
|
|
r = subprocess.run(
|
|
[LARK, 'api', 'GET', f'/open-apis/drive/v1/medias/{token}/download', '-o', filename],
|
|
capture_output=True, timeout=timeout, cwd=os.path.dirname(local_path)
|
|
)
|
|
return r.returncode == 0 and os.path.isfile(local_path)
|
|
except: return False
|
|
|
|
def resolve_wiki(node_id):
|
|
data = lark_api(f'/open-apis/wiki/v2/spaces/get_node?token={node_id}')
|
|
if data and data.get('code') == 0:
|
|
node = data.get('data', {}).get('node', {})
|
|
return node.get('obj_token', ''), node.get('title', '')
|
|
return '', ''
|
|
|
|
def extract_text(text_obj):
|
|
if not text_obj: return ''
|
|
parts = []
|
|
for elem in text_obj.get('elements', []):
|
|
if 'text_run' in elem:
|
|
content = elem['text_run'].get('content', '')
|
|
style = elem['text_run'].get('text_element_style', {})
|
|
if style.get('bold'): content = f'**{content}**'
|
|
if style.get('italic'): content = f'*{content}*'
|
|
parts.append(content)
|
|
elif 'mention_user' in elem:
|
|
parts.append(f'@{elem["mention_user"].get("user_id", "?")}')
|
|
return ''.join(parts).strip()
|
|
|
|
def blocks_to_markdown(blocks, doc_id, doc_url):
|
|
lines = []
|
|
image_map = {}
|
|
block_map = {b['block_id']: b for b in blocks}
|
|
root_blocks = [b for b in blocks if b.get('parent_id') == doc_id or not b.get('parent_id')]
|
|
|
|
def process(b, depth=0):
|
|
bt = b.get('block_type', 0)
|
|
children = b.get('children', [])
|
|
|
|
if bt == 2:
|
|
t = extract_text(b.get('text', {}))
|
|
if t: lines.append(t)
|
|
elif 3 <= bt <= 9:
|
|
t = extract_text(b.get(f'heading{bt-2}', {}) or b.get('text', {}))
|
|
if t: lines.append(f'{"#"*(bt-2)} {t}')
|
|
elif bt == 10:
|
|
t = extract_text(b.get('bullet', {}) or b.get('text', {}))
|
|
if t: lines.append(f'{" "*depth}- {t}')
|
|
elif bt == 11:
|
|
t = extract_text(b.get('ordered', {}) or b.get('text', {}))
|
|
if t: lines.append(f'{" "*depth}1. {t}')
|
|
elif bt == 12:
|
|
if 'bullet' in b:
|
|
t = extract_text(b.get('bullet', {}))
|
|
if t: lines.append(f'{" "*depth}- {t}')
|
|
else:
|
|
t = extract_text(b.get('code', {}) or b.get('text', {}))
|
|
if t: lines.append(f'```\n{t}\n```')
|
|
elif bt == 13:
|
|
t = extract_text(b.get('quote', {}) or b.get('text', {}))
|
|
if t: lines.append(f'> {t}')
|
|
elif bt == 14:
|
|
t = extract_text(b.get('todo', {}) or b.get('text', {}))
|
|
if t: lines.append(f'- [ ] {t}')
|
|
elif bt == 19:
|
|
if 'callout' in b:
|
|
emoji = b.get('callout', {}).get('emoji_id', '')
|
|
if emoji: lines.append(f'\n> :{emoji}:')
|
|
else:
|
|
lines.append('---')
|
|
elif bt == 27: # sheet = 图片
|
|
img = b.get('image', {})
|
|
token = img.get('token', '')
|
|
if token:
|
|
local_name = f'{doc_id}_{token[:12]}.png'
|
|
local_path = os.path.join(IMAGES_DIR, local_name)
|
|
image_map[token] = local_path
|
|
lines.append(f'')
|
|
elif bt == 30: # view = 嵌入表格
|
|
sheet_token = b.get('sheet', {}).get('token', '')
|
|
if sheet_token:
|
|
lines.append(f'\n> [嵌入电子表格: {sheet_token[:20]}...]\n')
|
|
elif bt == 28:
|
|
lines.append('\n| (嵌入表格) |')
|
|
elif bt in (21, 22, 24, 25, 31, 34): # 容器
|
|
pass
|
|
else:
|
|
for key in ['text', 'bullet', 'heading1', 'heading2', 'heading3', 'quote', 'code', 'todo']:
|
|
if key in b:
|
|
t = extract_text(b[key])
|
|
if t: lines.append(t)
|
|
break
|
|
|
|
for cid in children:
|
|
if cid in block_map:
|
|
process(block_map[cid], depth + 1)
|
|
|
|
for b in root_blocks:
|
|
process(b, depth=0)
|
|
|
|
header = [f'# {doc_id}\n', f'Source: {doc_url}\n', '']
|
|
return '\n'.join(header + lines), image_map
|
|
|
|
def fetch_document(doc_id, doc_type='docx', doc_url='', title=''):
|
|
real_id = doc_id
|
|
if doc_type == 'wiki':
|
|
real_id, wiki_title = resolve_wiki(doc_id)
|
|
if not real_id: return None, {}, f'wiki解析失败: {doc_id}'
|
|
if wiki_title and not title: title = wiki_title
|
|
|
|
data = lark_api(f'/open-apis/docx/v1/documents/{real_id}/blocks')
|
|
if not data or data.get('code') != 0:
|
|
return None, {}, f'API失败'
|
|
|
|
blocks = data.get('data', {}).get('items', [])
|
|
if not blocks: return None, {}, '空文档'
|
|
|
|
md, image_map = blocks_to_markdown(blocks, doc_id, doc_url)
|
|
|
|
downloaded = 0
|
|
for token, local_path in image_map.items():
|
|
if download_media(token, local_path):
|
|
downloaded += 1
|
|
time.sleep(0.15)
|
|
|
|
stats = {
|
|
'total_blocks': len(blocks),
|
|
'text_blocks': len([b for b in blocks if b.get('block_type') in (2,3,4,5,6,7,8,9,10,11,13)]),
|
|
'code_blocks': len([b for b in blocks if b.get('block_type') == 12]),
|
|
'image_blocks': len([b for b in blocks if b.get('block_type') == 27]),
|
|
'view_blocks': len([b for b in blocks if b.get('block_type') == 30]),
|
|
'images_downloaded': downloaded,
|
|
'content_length': len(md),
|
|
}
|
|
return md, stats, None
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--rebuild', action='store_true')
|
|
p.add_argument('--doc', type=str)
|
|
args = p.parse_args()
|
|
|
|
os.makedirs(DOCS_DIR, exist_ok=True)
|
|
os.makedirs(IMAGES_DIR, exist_ok=True)
|
|
|
|
if args.doc:
|
|
md, stats, err = fetch_document(args.doc, doc_url=f'https://feishu.cn/docx/{args.doc}')
|
|
if err: print(f'ERROR: {err}'); sys.exit(1)
|
|
with open(os.path.join(DOCS_DIR, f'{args.doc}.md'), 'w', encoding='utf-8') as f: f.write(md)
|
|
print(f'Done: {stats}')
|
|
return
|
|
|
|
links_path = os.path.join(LINKS_DIR, 'all_feishu_links.json')
|
|
if not os.path.isfile(links_path): print('ERROR: 先运行 step2'); sys.exit(1)
|
|
with open(links_path, 'r', encoding='utf-8') as f: all_links = json.load(f).get('links', [])
|
|
|
|
content_path = os.path.join(LINKS_DIR, 'all_feishu_content.json')
|
|
existing = {}
|
|
if os.path.isfile(content_path) and not args.rebuild:
|
|
with open(content_path, 'r', encoding='utf-8') as f:
|
|
for d in json.load(f).get('documents', []): existing[d.get('doc_id', '')] = d
|
|
|
|
total = len(all_links)
|
|
new = skip = err = 0
|
|
print(f'总链接: {total}, 已有: {len(existing)}, 模式: {"重建" if args.rebuild else "增量"}\n')
|
|
|
|
for i, link in enumerate(all_links):
|
|
doc_id = link.get('doc_id', '')
|
|
if not doc_id: continue
|
|
|
|
if not args.rebuild and doc_id in existing:
|
|
old = existing[doc_id]
|
|
if old.get('fetched', False) and old.get('content_length', 0) > 200 and old.get('images_downloaded', 0) > 0:
|
|
skip += 1
|
|
continue
|
|
|
|
doc_type = link.get('type', 'docx')
|
|
url = link.get('url', '')
|
|
sender = link.get('sender', '?')
|
|
group = link.get('group', '?')
|
|
|
|
print(f'[{i+1}/{total}] {doc_id} ({sender})...', end=' ', flush=True)
|
|
md, stats, error = fetch_document(doc_id, doc_type, url)
|
|
|
|
if error:
|
|
print(f'FAIL: {error}'); err += 1; continue
|
|
|
|
with open(os.path.join(DOCS_DIR, f'{doc_id}.md'), 'w', encoding='utf-8') as f: f.write(md)
|
|
|
|
title = ''
|
|
for line in md.split('\n'):
|
|
if line.startswith('# ') and len(line) > 2: title = line[2:].strip(); break
|
|
|
|
existing[doc_id] = {
|
|
'url': url, 'doc_id': doc_id, 'title': title,
|
|
'sender': sender, 'group': group, 'time': link.get('time', ''),
|
|
'content_length': stats['content_length'],
|
|
'total_blocks': stats['total_blocks'],
|
|
'image_blocks': stats['image_blocks'],
|
|
'images_downloaded': stats['images_downloaded'],
|
|
'fetched': True, 'fetched_at': datetime.now().isoformat(),
|
|
}
|
|
new += 1
|
|
print(f'OK ({stats["content_length"]}c, {stats["image_blocks"]}img, {stats["images_downloaded"]}dl)')
|
|
time.sleep(0.2)
|
|
|
|
with open(content_path, 'w', encoding='utf-8') as f:
|
|
json.dump({'date': datetime.now().strftime('%Y-%m-%d'), 'total': len(existing), 'documents': list(existing.values())}, f, ensure_ascii=False, indent=2)
|
|
|
|
imgs = len([f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')]) if os.path.isdir(IMAGES_DIR) else 0
|
|
print(f'\n完成! 新拉取:{new} 跳过:{skip} 失败:{err} 总计:{len(existing)} 图片:{imgs}')
|
|
|
|
if __name__ == '__main__': main()
|