"""飞书文档完整拉取器 - 处理所有 block 类型 + 图片下载""" import argparse, json, os, subprocess, sys, time from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR LARK = r'C:\Users\admin\AppData\Roaming\npm\node_modules\@larksuite\cli\bin\lark-cli.exe' IMAGES_DIR = os.path.join(OUTPUT_DIR, 'feishu-images') def lark_api(path, timeout=30): r = subprocess.run([LARK, 'api', 'GET', path, '--format', 'json'], capture_output=True, timeout=timeout) out = r.stdout.decode('utf-8', errors='replace') try: return json.loads(out) except: return None def download_media(token, local_path, timeout=30): """下载飞书媒体文件 - lark-cli 要求相对路径，需用 cwd""" if os.path.isfile(local_path): return True os.makedirs(os.path.dirname(local_path), exist_ok=True) filename = os.path.basename(local_path) try: r = subprocess.run( [LARK, 'api', 'GET', f'/open-apis/drive/v1/medias/{token}/download', '-o', filename], capture_output=True, timeout=timeout, cwd=os.path.dirname(local_path) ) return r.returncode == 0 and os.path.isfile(local_path) except: return False def resolve_wiki(node_id): data = lark_api(f'/open-apis/wiki/v2/spaces/get_node?token={node_id}') if data and data.get('code') == 0: node = data.get('data', {}).get('node', {}) return node.get('obj_token', ''), node.get('title', '') return '', '' def extract_text(text_obj): if not text_obj: return '' parts = [] for elem in text_obj.get('elements', []): if 'text_run' in elem: content = elem['text_run'].get('content', '') style = elem['text_run'].get('text_element_style', {}) if style.get('bold'): content = f'**{content}**' if style.get('italic'): content = f'*{content}*' parts.append(content) elif 'mention_user' in elem: parts.append(f'@{elem["mention_user"].get("user_id", "?")}') return ''.join(parts).strip() def blocks_to_markdown(blocks, doc_id, doc_url): lines = [] image_map = {} block_map = {b['block_id']: b for b in blocks} root_blocks = [b for b in blocks if b.get('parent_id') == doc_id or not b.get('parent_id')] def process(b, depth=0): bt = b.get('block_type', 0) children = b.get('children', []) if bt == 2: t = extract_text(b.get('text', {})) if t: lines.append(t) elif 3 <= bt <= 9: t = extract_text(b.get(f'heading{bt-2}', {}) or b.get('text', {})) if t: lines.append(f'{"#"*(bt-2)} {t}') elif bt == 10: t = extract_text(b.get('bullet', {}) or b.get('text', {})) if t: lines.append(f'{" "*depth}- {t}') elif bt == 11: t = extract_text(b.get('ordered', {}) or b.get('text', {})) if t: lines.append(f'{" "*depth}1. {t}') elif bt == 12: if 'bullet' in b: t = extract_text(b.get('bullet', {})) if t: lines.append(f'{" "*depth}- {t}') else: t = extract_text(b.get('code', {}) or b.get('text', {})) if t: lines.append(f'```\n{t}\n```') elif bt == 13: t = extract_text(b.get('quote', {}) or b.get('text', {})) if t: lines.append(f'> {t}') elif bt == 14: t = extract_text(b.get('todo', {}) or b.get('text', {})) if t: lines.append(f'- [ ] {t}') elif bt == 19: if 'callout' in b: emoji = b.get('callout', {}).get('emoji_id', '') if emoji: lines.append(f'\n> :{emoji}:') else: lines.append('---') elif bt == 27: # sheet = 图片 img = b.get('image', {}) token = img.get('token', '') if token: local_name = f'{doc_id}_{token[:12]}.png' local_path = os.path.join(IMAGES_DIR, local_name) image_map[token] = local_path lines.append(f'![图片](../feishu-images/{local_name})') elif bt == 30: # view = 嵌入表格 sheet_token = b.get('sheet', {}).get('token', '') if sheet_token: lines.append(f'\n> [嵌入电子表格: {sheet_token[:20]}...]\n') elif bt == 28: lines.append('\n| (嵌入表格) |') elif bt in (21, 22, 24, 25, 31, 34): # 容器 pass else: for key in ['text', 'bullet', 'heading1', 'heading2', 'heading3', 'quote', 'code', 'todo']: if key in b: t = extract_text(b[key]) if t: lines.append(t) break for cid in children: if cid in block_map: process(block_map[cid], depth + 1) for b in root_blocks: process(b, depth=0) header = [f'# {doc_id}\n', f'Source: {doc_url}\n', ''] return '\n'.join(header + lines), image_map def fetch_document(doc_id, doc_type='docx', doc_url='', title=''): real_id = doc_id if doc_type == 'wiki': real_id, wiki_title = resolve_wiki(doc_id) if not real_id: return None, {}, f'wiki解析失败: {doc_id}' if wiki_title and not title: title = wiki_title data = lark_api(f'/open-apis/docx/v1/documents/{real_id}/blocks') if not data or data.get('code') != 0: return None, {}, f'API失败' blocks = data.get('data', {}).get('items', []) if not blocks: return None, {}, '空文档' md, image_map = blocks_to_markdown(blocks, doc_id, doc_url) downloaded = 0 for token, local_path in image_map.items(): if download_media(token, local_path): downloaded += 1 time.sleep(0.15) stats = { 'total_blocks': len(blocks), 'text_blocks': len([b for b in blocks if b.get('block_type') in (2,3,4,5,6,7,8,9,10,11,13)]), 'code_blocks': len([b for b in blocks if b.get('block_type') == 12]), 'image_blocks': len([b for b in blocks if b.get('block_type') == 27]), 'view_blocks': len([b for b in blocks if b.get('block_type') == 30]), 'images_downloaded': downloaded, 'content_length': len(md), } return md, stats, None def main(): p = argparse.ArgumentParser() p.add_argument('--rebuild', action='store_true') p.add_argument('--doc', type=str) args = p.parse_args() os.makedirs(DOCS_DIR, exist_ok=True) os.makedirs(IMAGES_DIR, exist_ok=True) if args.doc: md, stats, err = fetch_document(args.doc, doc_url=f'https://feishu.cn/docx/{args.doc}') if err: print(f'ERROR: {err}'); sys.exit(1) with open(os.path.join(DOCS_DIR, f'{args.doc}.md'), 'w', encoding='utf-8') as f: f.write(md) print(f'Done: {stats}') return links_path = os.path.join(LINKS_DIR, 'all_feishu_links.json') if not os.path.isfile(links_path): print('ERROR: 先运行 step2'); sys.exit(1) with open(links_path, 'r', encoding='utf-8') as f: all_links = json.load(f).get('links', []) content_path = os.path.join(LINKS_DIR, 'all_feishu_content.json') existing = {} if os.path.isfile(content_path) and not args.rebuild: with open(content_path, 'r', encoding='utf-8') as f: for d in json.load(f).get('documents', []): existing[d.get('doc_id', '')] = d total = len(all_links) new = skip = err = 0 print(f'总链接: {total}, 已有: {len(existing)}, 模式: {"重建" if args.rebuild else "增量"}\n') for i, link in enumerate(all_links): doc_id = link.get('doc_id', '') if not doc_id: continue if not args.rebuild and doc_id in existing: old = existing[doc_id] if old.get('fetched', False) and old.get('content_length', 0) > 200 and old.get('images_downloaded', 0) > 0: skip += 1 continue doc_type = link.get('type', 'docx') url = link.get('url', '') sender = link.get('sender', '?') group = link.get('group', '?') print(f'[{i+1}/{total}] {doc_id} ({sender})...', end=' ', flush=True) md, stats, error = fetch_document(doc_id, doc_type, url) if error: print(f'FAIL: {error}'); err += 1; continue with open(os.path.join(DOCS_DIR, f'{doc_id}.md'), 'w', encoding='utf-8') as f: f.write(md) title = '' for line in md.split('\n'): if line.startswith('# ') and len(line) > 2: title = line[2:].strip(); break existing[doc_id] = { 'url': url, 'doc_id': doc_id, 'title': title, 'sender': sender, 'group': group, 'time': link.get('time', ''), 'content_length': stats['content_length'], 'total_blocks': stats['total_blocks'], 'image_blocks': stats['image_blocks'], 'images_downloaded': stats['images_downloaded'], 'fetched': True, 'fetched_at': datetime.now().isoformat(), } new += 1 print(f'OK ({stats["content_length"]}c, {stats["image_blocks"]}img, {stats["images_downloaded"]}dl)') time.sleep(0.2) with open(content_path, 'w', encoding='utf-8') as f: json.dump({'date': datetime.now().strftime('%Y-%m-%d'), 'total': len(existing), 'documents': list(existing.values())}, f, ensure_ascii=False, indent=2) imgs = len([f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')]) if os.path.isdir(IMAGES_DIR) else 0 print(f'\n完成! 新拉取:{new} 跳过:{skip} 失败:{err} 总计:{len(existing)} 图片:{imgs}') if __name__ == '__main__': main()