Evilom
fe5505343e
- 通过悟空(dws CLI)拉取dc战略问题研究院+创新组两个群的消息(377条) - 提取190个飞书链接、18个文件附件 - 下载HTML/MD/XLSX等报告文件到output/downloaded-files/ - 构建知识图谱(JSON+HTML可视化) - 生成Obsidian知识库(28个页面,7大主题) - 生成花园世界全量汇总报告 - 所有脚本路径改为相对路径,便于迁移
92 lines
3.1 KiB
Python
92 lines
3.1 KiB
Python
import json
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import time
|
|
import os
|
|
|
|
def extract_full(url):
|
|
try:
|
|
resp = requests.get(url, timeout=15, headers={'User-Agent': 'Mozilla/5.0'})
|
|
soup = BeautifulSoup(resp.text, 'html.parser')
|
|
|
|
# Remove scripts and styles
|
|
for s in soup(['script', 'style', 'nav', 'header', 'footer']):
|
|
s.decompose()
|
|
|
|
# Try to find the main content area
|
|
# Feishu docs usually have content in specific divs
|
|
content = ""
|
|
|
|
# Method 1: Look for doc-content or wiki-content
|
|
main_content = soup.find('div', {'class': lambda x: x and ('doc-content' in x or 'wiki-content' in x or 'document-content' in x) if x else False})
|
|
if main_content:
|
|
content = main_content.get_text(separator='\n', strip=True)
|
|
|
|
# Method 2: Look for article or main tag
|
|
if not content:
|
|
article = soup.find('article') or soup.find('main')
|
|
if article:
|
|
content = article.get_text(separator='\n', strip=True)
|
|
|
|
# Method 3: Get all text from body
|
|
if not content:
|
|
body = soup.find('body')
|
|
if body:
|
|
content = body.get_text(separator='\n', strip=True)
|
|
|
|
# Clean up the content
|
|
lines = content.split('\n')
|
|
cleaned_lines = []
|
|
for line in lines:
|
|
line = line.strip()
|
|
if line and len(line) > 1: # Skip empty lines and single chars
|
|
cleaned_lines.append(line)
|
|
|
|
return '\n'.join(cleaned_lines)
|
|
except Exception as e:
|
|
return 'Error: ' + str(e)
|
|
|
|
# Load links
|
|
with open('D:/desktop/feishu_docs_summary/all_feishu_links.json', 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
# Load existing progress
|
|
progress_file = 'D:/desktop/feishu_docs_summary/full_content_progress.json'
|
|
if os.path.exists(progress_file):
|
|
with open(progress_file, 'r', encoding='utf-8') as f:
|
|
progress = json.load(f)
|
|
else:
|
|
progress = {'read': [], 'docs': []}
|
|
|
|
# Get unread links
|
|
read_urls = set(progress['read'])
|
|
unread = [l for l in data['links'] if l['url'] not in read_urls]
|
|
print('Already read: ' + str(len(read_urls)))
|
|
print('Remaining: ' + str(len(unread)))
|
|
|
|
# Read batch of 5 (slower but more complete)
|
|
batch_size = 5
|
|
batch = unread[:batch_size]
|
|
print('Reading batch of ' + str(len(batch)) + '...')
|
|
|
|
for i, link in enumerate(batch, 1):
|
|
print(' [' + str(i) + '/' + str(len(batch)) + '] ' + link['sender'] + ' - ' + link['url'][:50])
|
|
content = extract_full(link['url'])
|
|
progress['read'].append(link['url'])
|
|
progress['docs'].append({
|
|
'group': link['group'],
|
|
'sender': link['sender'],
|
|
'time': link['time'],
|
|
'url': link['url'],
|
|
'content': content,
|
|
'content_length': len(content)
|
|
})
|
|
time.sleep(0.3)
|
|
|
|
# Save progress
|
|
with open(progress_file, 'w', encoding='utf-8') as f:
|
|
json.dump(progress, f, ensure_ascii=False, indent=2)
|
|
|
|
print('Total read: ' + str(len(progress['read'])))
|
|
print('Sample content length: ' + str(progress['docs'][-1]['content_length']) + ' chars')
|