import json
import requests
from bs4 import BeautifulSoup
import time
import os

def extract_full(url):
    try:
        resp = requests.get(url, timeout=15, headers={'User-Agent': 'Mozilla/5.0'})
        soup = BeautifulSoup(resp.text, 'html.parser')
        
        # Remove scripts and styles
        for s in soup(['script', 'style', 'nav', 'header', 'footer']):
            s.decompose()
        
        # Try to find the main content area
        # Feishu docs usually have content in specific divs
        content = ""
        
        # Method 1: Look for doc-content or wiki-content
        main_content = soup.find('div', {'class': lambda x: x and ('doc-content' in x or 'wiki-content' in x or 'document-content' in x) if x else False})
        if main_content:
            content = main_content.get_text(separator='\n', strip=True)
        
        # Method 2: Look for article or main tag
        if not content:
            article = soup.find('article') or soup.find('main')
            if article:
                content = article.get_text(separator='\n', strip=True)
        
        # Method 3: Get all text from body
        if not content:
            body = soup.find('body')
            if body:
                content = body.get_text(separator='\n', strip=True)
        
        # Clean up the content
        lines = content.split('\n')
        cleaned_lines = []
        for line in lines:
            line = line.strip()
            if line and len(line) > 1:  # Skip empty lines and single chars
                cleaned_lines.append(line)
        
        return '\n'.join(cleaned_lines)
    except Exception as e:
        return 'Error: ' + str(e)

# Load links
with open('D:/desktop/feishu_docs_summary/all_feishu_links.json', 'r', encoding='utf-8') as f:
    data = json.load(f)

# Load existing progress
progress_file = 'D:/desktop/feishu_docs_summary/full_content_progress.json'
if os.path.exists(progress_file):
    with open(progress_file, 'r', encoding='utf-8') as f:
        progress = json.load(f)
else:
    progress = {'read': [], 'docs': []}

# Get unread links
read_urls = set(progress['read'])
unread = [l for l in data['links'] if l['url'] not in read_urls]
print('Already read: ' + str(len(read_urls)))
print('Remaining: ' + str(len(unread)))

# Read batch of 5 (slower but more complete)
batch_size = 5
batch = unread[:batch_size]
print('Reading batch of ' + str(len(batch)) + '...')

for i, link in enumerate(batch, 1):
    print('  [' + str(i) + '/' + str(len(batch)) + '] ' + link['sender'] + ' - ' + link['url'][:50])
    content = extract_full(link['url'])
    progress['read'].append(link['url'])
    progress['docs'].append({
        'group': link['group'],
        'sender': link['sender'],
        'time': link['time'],
        'url': link['url'],
        'content': content,
        'content_length': len(content)
    })
    time.sleep(0.3)

# Save progress
with open(progress_file, 'w', encoding='utf-8') as f:
    json.dump(progress, f, ensure_ascii=False, indent=2)

print('Total read: ' + str(len(progress['read'])))
print('Sample content length: ' + str(progress['docs'][-1]['content_length']) + ' chars')