feat：飞书文档补全
This commit is contained in:
Evilom
@@ -0,0 +1,235 @@
|
||||
"""飞书文档完整拉取器 - 处理所有 block 类型 + 图片下载"""
|
||||
|
||||
import argparse, json, os, subprocess, sys, time
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR
|
||||
|
||||
LARK = r'C:\Users\admin\AppData\Roaming\npm\node_modules\@larksuite\cli\bin\lark-cli.exe'
|
||||
IMAGES_DIR = os.path.join(OUTPUT_DIR, 'feishu-images')
|
||||
|
||||
def lark_api(path, timeout=30):
|
||||
r = subprocess.run([LARK, 'api', 'GET', path, '--format', 'json'], capture_output=True, timeout=timeout)
|
||||
out = r.stdout.decode('utf-8', errors='replace')
|
||||
try: return json.loads(out)
|
||||
except: return None
|
||||
|
||||
def download_media(token, local_path, timeout=30):
|
||||
"""下载飞书媒体文件 - lark-cli 要求相对路径，需用 cwd"""
|
||||
if os.path.isfile(local_path): return True
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
filename = os.path.basename(local_path)
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[LARK, 'api', 'GET', f'/open-apis/drive/v1/medias/{token}/download', '-o', filename],
|
||||
capture_output=True, timeout=timeout, cwd=os.path.dirname(local_path)
|
||||
)
|
||||
return r.returncode == 0 and os.path.isfile(local_path)
|
||||
except: return False
|
||||
|
||||
def resolve_wiki(node_id):
|
||||
data = lark_api(f'/open-apis/wiki/v2/spaces/get_node?token={node_id}')
|
||||
if data and data.get('code') == 0:
|
||||
node = data.get('data', {}).get('node', {})
|
||||
return node.get('obj_token', ''), node.get('title', '')
|
||||
return '', ''
|
||||
|
||||
def extract_text(text_obj):
|
||||
if not text_obj: return ''
|
||||
parts = []
|
||||
for elem in text_obj.get('elements', []):
|
||||
if 'text_run' in elem:
|
||||
content = elem['text_run'].get('content', '')
|
||||
style = elem['text_run'].get('text_element_style', {})
|
||||
if style.get('bold'): content = f'**{content}**'
|
||||
if style.get('italic'): content = f'*{content}*'
|
||||
parts.append(content)
|
||||
elif 'mention_user' in elem:
|
||||
parts.append(f'@{elem["mention_user"].get("user_id", "?")}')
|
||||
return ''.join(parts).strip()
|
||||
|
||||
def blocks_to_markdown(blocks, doc_id, doc_url):
|
||||
lines = []
|
||||
image_map = {}
|
||||
block_map = {b['block_id']: b for b in blocks}
|
||||
root_blocks = [b for b in blocks if b.get('parent_id') == doc_id or not b.get('parent_id')]
|
||||
|
||||
def process(b, depth=0):
|
||||
bt = b.get('block_type', 0)
|
||||
children = b.get('children', [])
|
||||
|
||||
if bt == 2:
|
||||
t = extract_text(b.get('text', {}))
|
||||
if t: lines.append(t)
|
||||
elif 3 <= bt <= 9:
|
||||
t = extract_text(b.get(f'heading{bt-2}', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{"#"*(bt-2)} {t}')
|
||||
elif bt == 10:
|
||||
t = extract_text(b.get('bullet', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{" "*depth}- {t}')
|
||||
elif bt == 11:
|
||||
t = extract_text(b.get('ordered', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{" "*depth}1. {t}')
|
||||
elif bt == 12:
|
||||
if 'bullet' in b:
|
||||
t = extract_text(b.get('bullet', {}))
|
||||
if t: lines.append(f'{" "*depth}- {t}')
|
||||
else:
|
||||
t = extract_text(b.get('code', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'```\n{t}\n```')
|
||||
elif bt == 13:
|
||||
t = extract_text(b.get('quote', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'> {t}')
|
||||
elif bt == 14:
|
||||
t = extract_text(b.get('todo', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'- [ ] {t}')
|
||||
elif bt == 19:
|
||||
if 'callout' in b:
|
||||
emoji = b.get('callout', {}).get('emoji_id', '')
|
||||
if emoji: lines.append(f'\n> :{emoji}:')
|
||||
else:
|
||||
lines.append('---')
|
||||
elif bt == 27: # sheet = 图片
|
||||
img = b.get('image', {})
|
||||
token = img.get('token', '')
|
||||
if token:
|
||||
local_name = f'{doc_id}_{token[:12]}.png'
|
||||
local_path = os.path.join(IMAGES_DIR, local_name)
|
||||
image_map[token] = local_path
|
||||
lines.append(f'')
|
||||
elif bt == 30: # view = 嵌入表格
|
||||
sheet_token = b.get('sheet', {}).get('token', '')
|
||||
if sheet_token:
|
||||
lines.append(f'\n> [嵌入电子表格: {sheet_token[:20]}...]\n')
|
||||
elif bt == 28:
|
||||
lines.append('\n| (嵌入表格) |')
|
||||
elif bt in (21, 22, 24, 25, 31, 34): # 容器
|
||||
pass
|
||||
else:
|
||||
for key in ['text', 'bullet', 'heading1', 'heading2', 'heading3', 'quote', 'code', 'todo']:
|
||||
if key in b:
|
||||
t = extract_text(b[key])
|
||||
if t: lines.append(t)
|
||||
break
|
||||
|
||||
for cid in children:
|
||||
if cid in block_map:
|
||||
process(block_map[cid], depth + 1)
|
||||
|
||||
for b in root_blocks:
|
||||
process(b, depth=0)
|
||||
|
||||
header = [f'# {doc_id}\n', f'Source: {doc_url}\n', '']
|
||||
return '\n'.join(header + lines), image_map
|
||||
|
||||
def fetch_document(doc_id, doc_type='docx', doc_url='', title=''):
|
||||
real_id = doc_id
|
||||
if doc_type == 'wiki':
|
||||
real_id, wiki_title = resolve_wiki(doc_id)
|
||||
if not real_id: return None, {}, f'wiki解析失败: {doc_id}'
|
||||
if wiki_title and not title: title = wiki_title
|
||||
|
||||
data = lark_api(f'/open-apis/docx/v1/documents/{real_id}/blocks')
|
||||
if not data or data.get('code') != 0:
|
||||
return None, {}, f'API失败'
|
||||
|
||||
blocks = data.get('data', {}).get('items', [])
|
||||
if not blocks: return None, {}, '空文档'
|
||||
|
||||
md, image_map = blocks_to_markdown(blocks, doc_id, doc_url)
|
||||
|
||||
downloaded = 0
|
||||
for token, local_path in image_map.items():
|
||||
if download_media(token, local_path):
|
||||
downloaded += 1
|
||||
time.sleep(0.15)
|
||||
|
||||
stats = {
|
||||
'total_blocks': len(blocks),
|
||||
'text_blocks': len([b for b in blocks if b.get('block_type') in (2,3,4,5,6,7,8,9,10,11,13)]),
|
||||
'code_blocks': len([b for b in blocks if b.get('block_type') == 12]),
|
||||
'image_blocks': len([b for b in blocks if b.get('block_type') == 27]),
|
||||
'view_blocks': len([b for b in blocks if b.get('block_type') == 30]),
|
||||
'images_downloaded': downloaded,
|
||||
'content_length': len(md),
|
||||
}
|
||||
return md, stats, None
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--rebuild', action='store_true')
|
||||
p.add_argument('--doc', type=str)
|
||||
args = p.parse_args()
|
||||
|
||||
os.makedirs(DOCS_DIR, exist_ok=True)
|
||||
os.makedirs(IMAGES_DIR, exist_ok=True)
|
||||
|
||||
if args.doc:
|
||||
md, stats, err = fetch_document(args.doc, doc_url=f'https://feishu.cn/docx/{args.doc}')
|
||||
if err: print(f'ERROR: {err}'); sys.exit(1)
|
||||
with open(os.path.join(DOCS_DIR, f'{args.doc}.md'), 'w', encoding='utf-8') as f: f.write(md)
|
||||
print(f'Done: {stats}')
|
||||
return
|
||||
|
||||
links_path = os.path.join(LINKS_DIR, 'all_feishu_links.json')
|
||||
if not os.path.isfile(links_path): print('ERROR: 先运行 step2'); sys.exit(1)
|
||||
with open(links_path, 'r', encoding='utf-8') as f: all_links = json.load(f).get('links', [])
|
||||
|
||||
content_path = os.path.join(LINKS_DIR, 'all_feishu_content.json')
|
||||
existing = {}
|
||||
if os.path.isfile(content_path) and not args.rebuild:
|
||||
with open(content_path, 'r', encoding='utf-8') as f:
|
||||
for d in json.load(f).get('documents', []): existing[d.get('doc_id', '')] = d
|
||||
|
||||
total = len(all_links)
|
||||
new = skip = err = 0
|
||||
print(f'总链接: {total}, 已有: {len(existing)}, 模式: {"重建" if args.rebuild else "增量"}\n')
|
||||
|
||||
for i, link in enumerate(all_links):
|
||||
doc_id = link.get('doc_id', '')
|
||||
if not doc_id: continue
|
||||
|
||||
if not args.rebuild and doc_id in existing:
|
||||
old = existing[doc_id]
|
||||
if old.get('fetched', False) and old.get('content_length', 0) > 200 and old.get('images_downloaded', 0) > 0:
|
||||
skip += 1
|
||||
continue
|
||||
|
||||
doc_type = link.get('type', 'docx')
|
||||
url = link.get('url', '')
|
||||
sender = link.get('sender', '?')
|
||||
group = link.get('group', '?')
|
||||
|
||||
print(f'[{i+1}/{total}] {doc_id} ({sender})...', end=' ', flush=True)
|
||||
md, stats, error = fetch_document(doc_id, doc_type, url)
|
||||
|
||||
if error:
|
||||
print(f'FAIL: {error}'); err += 1; continue
|
||||
|
||||
with open(os.path.join(DOCS_DIR, f'{doc_id}.md'), 'w', encoding='utf-8') as f: f.write(md)
|
||||
|
||||
title = ''
|
||||
for line in md.split('\n'):
|
||||
if line.startswith('# ') and len(line) > 2: title = line[2:].strip(); break
|
||||
|
||||
existing[doc_id] = {
|
||||
'url': url, 'doc_id': doc_id, 'title': title,
|
||||
'sender': sender, 'group': group, 'time': link.get('time', ''),
|
||||
'content_length': stats['content_length'],
|
||||
'total_blocks': stats['total_blocks'],
|
||||
'image_blocks': stats['image_blocks'],
|
||||
'images_downloaded': stats['images_downloaded'],
|
||||
'fetched': True, 'fetched_at': datetime.now().isoformat(),
|
||||
}
|
||||
new += 1
|
||||
print(f'OK ({stats["content_length"]}c, {stats["image_blocks"]}img, {stats["images_downloaded"]}dl)')
|
||||
time.sleep(0.2)
|
||||
|
||||
with open(content_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({'date': datetime.now().strftime('%Y-%m-%d'), 'total': len(existing), 'documents': list(existing.values())}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
imgs = len([f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')]) if os.path.isdir(IMAGES_DIR) else 0
|
||||
print(f'\n完成! 新拉取:{new} 跳过:{skip} 失败:{err} 总计:{len(existing)} 图片:{imgs}')
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,197 @@
|
||||
"""修复 wiki 解析 + 重拉失败的文档"""
|
||||
import json, os, subprocess, sys, time
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from paths import LINKS_DIR, DOCS_DIR, OUTPUT_DIR
|
||||
|
||||
LARK = r'C:\Users\admin\AppData\Roaming\npm\node_modules\@larksuite\cli\bin\lark-cli.exe'
|
||||
IMAGES_DIR = os.path.join(OUTPUT_DIR, 'feishu-images')
|
||||
|
||||
def lark_api(path, params=None, timeout=30):
|
||||
cmd = [LARK, 'api', 'GET', path, '--format', 'json']
|
||||
if params: cmd += ['--params', json.dumps(params)]
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=timeout)
|
||||
out = r.stdout.decode('utf-8', errors='replace')
|
||||
try: return json.loads(out)
|
||||
except: return None
|
||||
|
||||
def download_media(token, local_path, timeout=30):
|
||||
if os.path.isfile(local_path): return True
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
filename = os.path.basename(local_path)
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[LARK, 'api', 'GET', f'/open-apis/drive/v1/medias/{token}/download', '-o', filename],
|
||||
capture_output=True, timeout=timeout, cwd=os.path.dirname(local_path)
|
||||
)
|
||||
return r.returncode == 0 and os.path.isfile(local_path)
|
||||
except: return False
|
||||
|
||||
def resolve_wiki(node_id):
|
||||
data = lark_api('/open-apis/wiki/v2/spaces/get_node', params={'token': node_id})
|
||||
if data and data.get('code') == 0:
|
||||
node = data.get('data', {}).get('node', {})
|
||||
return node.get('obj_token', ''), node.get('title', '')
|
||||
return '', ''
|
||||
|
||||
def extract_text(text_obj):
|
||||
if not text_obj: return ''
|
||||
parts = []
|
||||
for elem in text_obj.get('elements', []):
|
||||
if 'text_run' in elem:
|
||||
content = elem['text_run'].get('content', '')
|
||||
style = elem['text_run'].get('text_element_style', {})
|
||||
if style.get('bold'): content = f'**{content}**'
|
||||
if style.get('italic'): content = f'*{content}*'
|
||||
parts.append(content)
|
||||
elif 'mention_user' in elem:
|
||||
parts.append(f'@{elem["mention_user"].get("user_id", "?")}')
|
||||
return ''.join(parts).strip()
|
||||
|
||||
def blocks_to_markdown(blocks, doc_id, doc_url):
|
||||
lines = []
|
||||
image_map = {}
|
||||
block_map = {b['block_id']: b for b in blocks}
|
||||
root_blocks = [b for b in blocks if b.get('parent_id') == doc_id or not b.get('parent_id')]
|
||||
|
||||
def process(b, depth=0):
|
||||
bt = b.get('block_type', 0)
|
||||
children = b.get('children', [])
|
||||
|
||||
if bt == 2:
|
||||
t = extract_text(b.get('text', {}))
|
||||
if t: lines.append(t)
|
||||
elif 3 <= bt <= 9:
|
||||
t = extract_text(b.get(f'heading{bt-2}', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{"#"*(bt-2)} {t}')
|
||||
elif bt == 10:
|
||||
t = extract_text(b.get('bullet', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{" "*depth}- {t}')
|
||||
elif bt == 11:
|
||||
t = extract_text(b.get('ordered', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'{" "*depth}1. {t}')
|
||||
elif bt == 12:
|
||||
if 'bullet' in b:
|
||||
t = extract_text(b.get('bullet', {}))
|
||||
if t: lines.append(f'{" "*depth}- {t}')
|
||||
else:
|
||||
t = extract_text(b.get('code', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'```\n{t}\n```')
|
||||
elif bt == 13:
|
||||
t = extract_text(b.get('quote', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'> {t}')
|
||||
elif bt == 14:
|
||||
t = extract_text(b.get('todo', {}) or b.get('text', {}))
|
||||
if t: lines.append(f'- [ ] {t}')
|
||||
elif bt == 19:
|
||||
if 'callout' in b:
|
||||
emoji = b.get('callout', {}).get('emoji_id', '')
|
||||
if emoji: lines.append(f'\n> :{emoji}:')
|
||||
else: lines.append('---')
|
||||
elif bt == 27:
|
||||
img = b.get('image', {})
|
||||
token = img.get('token', '')
|
||||
if token:
|
||||
local_name = f'{doc_id}_{token[:12]}.png'
|
||||
local_path = os.path.join(IMAGES_DIR, local_name)
|
||||
image_map[token] = local_path
|
||||
lines.append(f'')
|
||||
elif bt == 30:
|
||||
sheet_token = b.get('sheet', {}).get('token', '')
|
||||
if sheet_token: lines.append(f'\n> [嵌入电子表格: {sheet_token[:20]}...]\n')
|
||||
elif bt == 28:
|
||||
lines.append('\n| (嵌入表格) |')
|
||||
elif bt in (21, 22, 24, 25, 31, 34): pass
|
||||
else:
|
||||
for key in ['text', 'bullet', 'heading1', 'heading2', 'heading3', 'quote', 'code', 'todo']:
|
||||
if key in b:
|
||||
t = extract_text(b[key])
|
||||
if t: lines.append(t)
|
||||
break
|
||||
|
||||
for cid in children:
|
||||
if cid in block_map: process(block_map[cid], depth + 1)
|
||||
|
||||
for b in root_blocks: process(b, depth=0)
|
||||
header = [f'# {doc_id}\n', f'Source: {doc_url}\n', '']
|
||||
return '\n'.join(header + lines), image_map
|
||||
|
||||
# 加载已有索引
|
||||
content_path = os.path.join(LINKS_DIR, 'all_feishu_content.json')
|
||||
with open(content_path, 'r', encoding='utf-8') as f:
|
||||
content_data = json.load(f)
|
||||
existing = {d['doc_id']: d for d in content_data.get('documents', [])}
|
||||
|
||||
# 找出需要重试的文档（fetched=False 或不存在的）
|
||||
links_path = os.path.join(LINKS_DIR, 'all_feishu_links.json')
|
||||
with open(links_path, 'r', encoding='utf-8') as f:
|
||||
all_links = json.load(f).get('links', [])
|
||||
|
||||
retry = []
|
||||
for link in all_links:
|
||||
did = link.get('doc_id', '')
|
||||
if not did: continue
|
||||
old = existing.get(did, {})
|
||||
if not old.get('fetched', False) or old.get('content_length', 0) <= 200:
|
||||
retry.append(link)
|
||||
|
||||
print(f'需要重试: {len(retry)} 篇\n')
|
||||
|
||||
new = err = 0
|
||||
for i, link in enumerate(retry):
|
||||
doc_id = link.get('doc_id', '')
|
||||
doc_type = link.get('type', 'docx')
|
||||
url = link.get('url', '')
|
||||
sender = link.get('sender', '?')
|
||||
|
||||
print(f'[{i+1}/{len(retry)}] {doc_id} ({sender})...', end=' ', flush=True)
|
||||
|
||||
real_id = doc_id
|
||||
title = ''
|
||||
if doc_type == 'wiki':
|
||||
real_id, wiki_title = resolve_wiki(doc_id)
|
||||
if not real_id:
|
||||
print('wiki FAIL'); err += 1; continue
|
||||
title = wiki_title
|
||||
|
||||
data = lark_api(f'/open-apis/docx/v1/documents/{real_id}/blocks')
|
||||
if not data or data.get('code') != 0:
|
||||
print('API FAIL'); err += 1; continue
|
||||
|
||||
blocks = data.get('data', {}).get('items', [])
|
||||
if not blocks:
|
||||
print('空'); err += 1; continue
|
||||
|
||||
md, image_map = blocks_to_markdown(blocks, doc_id, url)
|
||||
|
||||
downloaded = 0
|
||||
for token, local_path in image_map.items():
|
||||
if download_media(token, local_path): downloaded += 1
|
||||
time.sleep(0.1)
|
||||
|
||||
with open(os.path.join(DOCS_DIR, f'{doc_id}.md'), 'w', encoding='utf-8') as f: f.write(md)
|
||||
|
||||
if not title:
|
||||
for line in md.split('\n'):
|
||||
if line.startswith('# ') and len(line) > 2: title = line[2:].strip(); break
|
||||
|
||||
existing[doc_id] = {
|
||||
'url': url, 'doc_id': doc_id, 'title': title,
|
||||
'sender': sender, 'group': link.get('group', ''), 'time': link.get('time', ''),
|
||||
'content_length': len(md), 'image_blocks': len(image_map),
|
||||
'images_downloaded': downloaded, 'fetched': True,
|
||||
'fetched_at': datetime.now().isoformat(),
|
||||
}
|
||||
new += 1
|
||||
print(f'OK ({len(md)}c, {len(image_map)}img, {downloaded}dl)')
|
||||
time.sleep(0.15)
|
||||
|
||||
# 保存索引
|
||||
content_data['documents'] = list(existing.values())
|
||||
content_data['total'] = len(existing)
|
||||
with open(content_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(content_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
imgs = len([f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')]) if os.path.isdir(IMAGES_DIR) else 0
|
||||
print(f'\n完成! 新拉取:{new} 失败:{err} 总计:{len(existing)} 图片:{imgs}')
|
||||
Reference in New Issue
Block a user