refactor: 清理项目结构，移除冗余文件
- 删除32个根目录中间文件（json/md/py/ps1/js/csv） - 删除docs/目录（内容乱码不可用） - 删除4个一次性工具脚本（find_garden/garden_related/read_all_feishu/read_full_batch） - 删除1个中间版本报告（feishu_summary_report.md） - 移动工作流总结.html到output/downloaded-files/ - 简化.gitignore为通用规则 - 更新README.md反映清理后的结构 清理后根目录仅保留.gitignore和README.md，结构清晰。
This commit is contained in:
Evilom
@@ -1,28 +0,0 @@
|
||||
import sys, io, json, re
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
|
||||
|
||||
import os
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
|
||||
|
||||
with open(os.path.join(PROJECT_ROOT, "data", "raw-messages", "all_messages_combined.json"), "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Find all messages mentioning 花园世界 or garden
|
||||
garden_msgs = []
|
||||
for group_name, msgs in data.items():
|
||||
for msg in msgs:
|
||||
content = msg.get("content", "")
|
||||
if "花园" in content or "garden" in content.lower() or "GOS" in content or "gos" in content.lower() or "花灵" in content or "花材" in content or "花种" in content or "种花" in content or "麟贝" in content:
|
||||
garden_msgs.append({
|
||||
"sender": msg.get("sender",""),
|
||||
"time": msg.get("createTime",""),
|
||||
"group": "dc" if group_name == "dc" else "cx",
|
||||
"content": content[:300]
|
||||
})
|
||||
|
||||
print("=== 花园世界相关消息: %d 条 ===" % len(garden_msgs))
|
||||
for m in sorted(garden_msgs, key=lambda x: x["time"]):
|
||||
print("[%s] %s (%s): %s" % (m["time"][:10], m["sender"], m["group"], m["content"][:150]))
|
||||
print()
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
import sys, io, json, re
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
|
||||
|
||||
import os
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
|
||||
|
||||
with open(os.path.join(PROJECT_ROOT, "data", "raw-messages", "all_messages_combined.json"), "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
feishu_re = re.compile(r"https?://[a-zA-Z0-9.-]+\.feishu\.cn/(?:wiki|docx)/[A-Za-z0-9]+")
|
||||
|
||||
# Find feishu links shared around the same time as garden world discussions (05-25 to 06-02)
|
||||
print("=== 05-25~06-02 期间 dc群所有飞书链接 ===")
|
||||
for msg in data["dc"]:
|
||||
t = msg.get("createTime", "")
|
||||
if t >= "2026-05-25" and t <= "2026-06-02":
|
||||
content = msg.get("content", "")
|
||||
links = feishu_re.findall(content)
|
||||
if links:
|
||||
sender = msg.get("sender", "")
|
||||
for link in links:
|
||||
print(" [%s] %s: %s" % (t[:10], sender, link))
|
||||
|
||||
# Also find the chat content around 05-25 to 06-02 for garden world discussions
|
||||
print("\n=== 05-25~06-02 期间 dc群非链接讨论 ===")
|
||||
for msg in data["dc"]:
|
||||
t = msg.get("createTime", "")
|
||||
if t >= "2026-05-25" and t <= "2026-06-02":
|
||||
content = msg.get("content", "")
|
||||
if not feishu_re.search(content) and not content.startswith("[文件]") and not content.startswith("[图片") and not content.startswith("[视频") and len(content) > 10:
|
||||
sender = msg.get("sender", "")
|
||||
print(" [%s] %s: %s" % (t[:16], sender, content[:200]))
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""读取所有飞书链接并生成汇总报告"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
def extract_content(url):
|
||||
"""从飞书链接提取内容"""
|
||||
try:
|
||||
resp = requests.get(url, timeout=10, headers={'User-Agent': 'Mozilla/5.0'})
|
||||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||||
|
||||
# 获取标题
|
||||
title = soup.find('title')
|
||||
title_text = title.text.strip() if title else ''
|
||||
|
||||
# 移除脚本和样式
|
||||
for script in soup(['script', 'style']):
|
||||
script.decompose()
|
||||
|
||||
# 获取文本内容
|
||||
text = soup.get_text(strip=True)
|
||||
|
||||
# 提取关键信息
|
||||
# 查找日报内容（通常在特定区域）
|
||||
content_start = text.find('日报')
|
||||
if content_start == -1:
|
||||
content_start = text.find('工作')
|
||||
if content_start == -1:
|
||||
content_start = 0
|
||||
|
||||
content = text[content_start:content_start + 800]
|
||||
|
||||
return {
|
||||
'title': title_text,
|
||||
'content': content,
|
||||
'success': True
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
'title': '',
|
||||
'content': str(e),
|
||||
'success': False
|
||||
}
|
||||
|
||||
def main():
|
||||
# 读取链接
|
||||
with open('D:/desktop/feishu_docs_summary/all_feishu_links.json', 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
links = data.get('links', [])
|
||||
print(f'共 {len(links)} 个链接')
|
||||
|
||||
# 按组分类
|
||||
groups = {}
|
||||
for link in links:
|
||||
group = link['group']
|
||||
if group not in groups:
|
||||
groups[group] = []
|
||||
groups[group].append(link)
|
||||
|
||||
# 读取每个组的文档
|
||||
all_docs = []
|
||||
for group_name, group_links in groups.items():
|
||||
print(f'\n=== {group_name} ({len(group_links)} 个链接) ===')
|
||||
|
||||
for i, link in enumerate(group_links[:5], 1): # 每个组读取前5个
|
||||
print(f' [{i}/5] {link["sender"]} - {link["url"][:50]}...')
|
||||
result = extract_content(link['url'])
|
||||
result['sender'] = link['sender']
|
||||
result['time'] = link['time']
|
||||
result['group'] = group_name
|
||||
result['url'] = link['url']
|
||||
all_docs.append(result)
|
||||
time.sleep(0.3)
|
||||
|
||||
# 保存结果
|
||||
output = {
|
||||
'date': datetime.now().strftime('%Y-%m-%d'),
|
||||
'total_links': len(links),
|
||||
'docs_read': len(all_docs),
|
||||
'documents': all_docs
|
||||
}
|
||||
|
||||
with open('D:/desktop/feishu_docs_summary/feishu_docs_content.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 生成 Markdown 报告
|
||||
report = f'# 飞书文档汇总报告\n\n'
|
||||
report += f'**日期**: {datetime.now().strftime("%Y-%m-%d")}\n'
|
||||
report += f'**链接总数**: {len(links)}\n'
|
||||
report += f'**已读取**: {len(all_docs)}\n\n'
|
||||
|
||||
for group_name in groups.keys():
|
||||
group_docs = [d for d in all_docs if d['group'] == group_name]
|
||||
if group_docs:
|
||||
report += f'## {group_name}\n\n'
|
||||
for doc in group_docs:
|
||||
report += f'### {doc["sender"]} ({doc["time"]})\n'
|
||||
report += f'**链接**: {doc["url"]}\n\n'
|
||||
report += f'{doc["content"][:300]}\n\n'
|
||||
report += '---\n\n'
|
||||
|
||||
with open('D:/desktop/feishu_docs_summary/feishu_summary_report.md', 'w', encoding='utf-8') as f:
|
||||
f.write(report)
|
||||
|
||||
print(f'\n已保存到:')
|
||||
print(f' - feishu_docs_content.json')
|
||||
print(f' - feishu_summary_report.md')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,91 +0,0 @@
|
||||
import json
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import time
|
||||
import os
|
||||
|
||||
def extract_full(url):
|
||||
try:
|
||||
resp = requests.get(url, timeout=15, headers={'User-Agent': 'Mozilla/5.0'})
|
||||
soup = BeautifulSoup(resp.text, 'html.parser')
|
||||
|
||||
# Remove scripts and styles
|
||||
for s in soup(['script', 'style', 'nav', 'header', 'footer']):
|
||||
s.decompose()
|
||||
|
||||
# Try to find the main content area
|
||||
# Feishu docs usually have content in specific divs
|
||||
content = ""
|
||||
|
||||
# Method 1: Look for doc-content or wiki-content
|
||||
main_content = soup.find('div', {'class': lambda x: x and ('doc-content' in x or 'wiki-content' in x or 'document-content' in x) if x else False})
|
||||
if main_content:
|
||||
content = main_content.get_text(separator='\n', strip=True)
|
||||
|
||||
# Method 2: Look for article or main tag
|
||||
if not content:
|
||||
article = soup.find('article') or soup.find('main')
|
||||
if article:
|
||||
content = article.get_text(separator='\n', strip=True)
|
||||
|
||||
# Method 3: Get all text from body
|
||||
if not content:
|
||||
body = soup.find('body')
|
||||
if body:
|
||||
content = body.get_text(separator='\n', strip=True)
|
||||
|
||||
# Clean up the content
|
||||
lines = content.split('\n')
|
||||
cleaned_lines = []
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line and len(line) > 1: # Skip empty lines and single chars
|
||||
cleaned_lines.append(line)
|
||||
|
||||
return '\n'.join(cleaned_lines)
|
||||
except Exception as e:
|
||||
return 'Error: ' + str(e)
|
||||
|
||||
# Load links
|
||||
with open('D:/desktop/feishu_docs_summary/all_feishu_links.json', 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Load existing progress
|
||||
progress_file = 'D:/desktop/feishu_docs_summary/full_content_progress.json'
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, 'r', encoding='utf-8') as f:
|
||||
progress = json.load(f)
|
||||
else:
|
||||
progress = {'read': [], 'docs': []}
|
||||
|
||||
# Get unread links
|
||||
read_urls = set(progress['read'])
|
||||
unread = [l for l in data['links'] if l['url'] not in read_urls]
|
||||
print('Already read: ' + str(len(read_urls)))
|
||||
print('Remaining: ' + str(len(unread)))
|
||||
|
||||
# Read batch of 5 (slower but more complete)
|
||||
batch_size = 5
|
||||
batch = unread[:batch_size]
|
||||
print('Reading batch of ' + str(len(batch)) + '...')
|
||||
|
||||
for i, link in enumerate(batch, 1):
|
||||
print(' [' + str(i) + '/' + str(len(batch)) + '] ' + link['sender'] + ' - ' + link['url'][:50])
|
||||
content = extract_full(link['url'])
|
||||
progress['read'].append(link['url'])
|
||||
progress['docs'].append({
|
||||
'group': link['group'],
|
||||
'sender': link['sender'],
|
||||
'time': link['time'],
|
||||
'url': link['url'],
|
||||
'content': content,
|
||||
'content_length': len(content)
|
||||
})
|
||||
time.sleep(0.3)
|
||||
|
||||
# Save progress
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(progress, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print('Total read: ' + str(len(progress['read'])))
|
||||
print('Sample content length: ' + str(progress['docs'][-1]['content_length']) + ' chars')
|
||||
Reference in New Issue
Block a user