chore: collect latest dingTalk feishu docs

skills/dingtalk-feishu-collector/scripts/step3_fetch_feishu_docs.py

@@ -4,12 +4,29 @@ from datetime import datetime
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from paths import LINKS_DIR, DOCS_DIR, find_lark_cli
 
-def resolve_wiki(node_id, lark):
+def lark_api(lark, path, params=None, timeout=30):
+    cmd = [lark, "api", "GET", path, "--format", "json"]
+    if params:
+        cmd += ["--params", json.dumps(params, ensure_ascii=False)]
+    r = subprocess.run(cmd, capture_output=True, timeout=timeout)
+    out = r.stdout.decode("utf-8", errors="replace")
+    err = r.stderr.decode("utf-8", errors="replace")
+    if r.returncode != 0:
+        return None, err or out
     try:
-        r = subprocess.run([lark,"api","GET",f"/open-apis/wiki/v2/spaces/get_node?token={node_id}","--format","json"],capture_output=True,text=True,timeout=15)
-        if r.returncode==0: return json.loads(r.stdout).get("data",{}).get("node",{}).get("obj_token","")
-    except: pass
-    return ""
+        data = json.loads(out)
+    except Exception as e:
+        return None, f"JSON解析失败: {e}"
+    if data.get("ok") is False or data.get("code", 0) not in (0, None):
+        return None, data.get("error") or data
+    return data, None
+
+def resolve_wiki(node_id, lark):
+    data, _ = lark_api(lark, "/open-apis/wiki/v2/spaces/get_node", params={"token": node_id}, timeout=15)
+    if not data:
+        return "", ""
+    node = data.get("data",{}).get("node",{})
+    return node.get("obj_token",""), node.get("title","")
 
 def extract_text(obj):
     if not obj: return ""
@@ -31,14 +48,36 @@ def blocks_to_md(blocks, doc_id):
     return "\n\n".join(lines)
 
 def fetch_doc(doc_id, doc_type, lark):
-    real_id = resolve_wiki(doc_id, lark) if doc_type=="wiki" else doc_id
-    if not real_id: return f"# {doc_id}\n\n> wiki解析失败"
-    try:
-        r = subprocess.run([lark,"api","GET",f"/open-apis/docx/v1/documents/{real_id}/blocks","--format","json"],capture_output=True,text=True,timeout=30)
-        if r.returncode!=0: return f"# {doc_id}\n\n> 获取失败"
-        blocks = json.loads(r.stdout).get("data",{}).get("items",[])
-        return blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
-    except Exception as e: return f"# {doc_id}\n\n> {e}"
+    real_id, wiki_title = (doc_id, "")
+    if doc_type=="wiki":
+        real_id, wiki_title = resolve_wiki(doc_id, lark)
+    if not real_id:
+        return f"# {doc_id}\n\n> wiki解析失败"
+    data, error = lark_api(lark, f"/open-apis/docx/v1/documents/{real_id}/blocks", timeout=30)
+    if error:
+        return f"# {doc_id}\n\n> 获取失败: {error}"
+    blocks = data.get("data",{}).get("items",[]) if data else []
+    md = blocks_to_md(blocks, doc_id) if blocks else f"# {doc_id}\n\n> 空文档"
+    if wiki_title:
+        md = md.replace(f"# {doc_id}", f"# {wiki_title}", 1)
+    return md
+
+def should_fetch(doc_id, existing, docs_dir):
+    old = existing.get(doc_id)
+    if not old:
+        return True
+    if old.get("content_length", 0) <= 200:
+        return True
+    path = os.path.join(docs_dir, f"{doc_id}.md")
+    if not os.path.isfile(path):
+        return True
+    with open(path, "r", encoding="utf-8") as f:
+        head = f.read(500)
+    return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档"])
+
+def is_error_markdown(md):
+    head = md[:1000]
+    return any(marker in head for marker in ["获取失败", "wiki解析失败", "NoneType", "空文档", "forBidden", "permission denied"])
 
 def main():
     p = argparse.ArgumentParser(); p.add_argument("--incremental",action="store_true"); args = p.parse_args()
@@ -59,14 +98,19 @@ def main():
     new_count = 0
     for link in all_links:
         did = link.get("doc_id","")
-        if not did or did in existing: continue
+        if not did or not should_fetch(did, existing, DOCS_DIR): continue
         print(f"  {did} ({link.get('sender','?')})")
         md = fetch_doc(did, link.get("type","docx"), lark)
+        fetched = not is_error_markdown(md)
+        old = existing.get(did)
+        if old and old.get("content_length", 0) > 200 and not fetched:
+            print("    WARN: 获取失败，保留已有正文")
+            continue
         with open(os.path.join(DOCS_DIR,f"{did}.md"),"w",encoding="utf-8") as f: f.write(md)
         title = ""
         for line in md.split("\n"):
             if line.startswith("# "): title = line[2:].strip(); break
-        existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":True}
+        existing[did] = {"url":link.get("url",""),"doc_id":did,"title":title,"sender":link.get("sender",""),"group":link.get("group",""),"time":link.get("time",""),"content_length":len(md),"fetched":fetched}
         new_count += 1
 
     with open(content_path,"w",encoding="utf-8") as f: