Добавить тестовые каналы и Arweave-синхронизацию

2026-09-23 11:14:02 +03:00
parent 8b12760dde
commit ef707ec217
80 changed files with 15452 additions and 26 deletions
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import argparse,json,hashlib,mimetypes,time
+from pathlib import Path
+import requests
+try: import arweave
+except ImportError: arweave=None
+
+def atomic(path,obj):
+    p=Path(path); p.parent.mkdir(parents=True,exist_ok=True)
+    t=p.with_suffix(p.suffix+'.tmp'); t.write_text(json.dumps(obj,ensure_ascii=False,indent=2),encoding='utf8'); t.replace(p)
+
+def exists(gateway,txid,timeout=15):
+    url=f"{gateway.rstrip('/')}/{txid}"
+    try:
+        r=requests.head(url,allow_redirects=True,timeout=timeout)
+        if r.status_code in (200,204): return True
+        if r.status_code in (405,501):
+            r=requests.get(url,headers={'Range':'bytes=0-0'},timeout=timeout)
+            return r.status_code in (200,206)
+        return False
+    except requests.RequestException: return False
+
+def main():
+    ap=argparse.ArgumentParser(description='Prepare images for all SHiNE test channels')
+    ap.add_argument('--config',default='prepare.config.json'); a=ap.parse_args()
+    cfg=json.load(open(a.config,encoding='utf8')); base=Path(a.config).resolve().parent
+    rp=lambda s:(Path(s) if Path(s).is_absolute() else (base/s).resolve())
+    gateway=cfg['arweave'].get('gateway','https://arweave.net').rstrip('/')
+    statep=rp(cfg.get('upload_state_file','upload-state.json'))
+    state=json.loads(statep.read_text()) if statep.exists() else {'version':2,'files':{}}
+    wallet=None
+    def ensure_wallet():
+        nonlocal wallet
+        if wallet is None:
+            if arweave is None: raise SystemExit('pip install -r requirements.txt')
+            wallet=arweave.Wallet(str(rp(cfg['arweave']['wallet_file'])))
+        return wallet
+
+    summary={'channels':0,'posts':0,'images':0,'verified_existing':0,'uploaded':0}
+    for ch in cfg['channels']:
+        rawp=rp(ch['queue_file']); outp=rp(ch['prepared_queue_file'])
+        items=json.loads(rawp.read_text(encoding='utf8')); prepared=[]; summary['channels']+=1
+        for n,item in enumerate(items,1):
+            refs=[]
+            for rel in item.get('images',[]):
+                fp=(rawp.parent/rel).resolve(); data=fp.read_bytes(); sha=hashlib.sha256(data).hexdigest()
+                summary['images']+=1; rec=state['files'].get(sha); txid=None
+                if rec and exists(gateway,rec['id']):
+                    txid=rec['id']; summary['verified_existing']+=1
+                    print(f"[{ch['name']}:{n}] verified {fp.name} -> {txid}")
+                else:
+                    ctype=mimetypes.guess_type(fp.name)[0] or 'application/octet-stream'
+                    tx=arweave.Transaction(ensure_wallet(),data=data)
+                    tx.add_tag('Content-Type',ctype); tx.add_tag('App-Name','SHiNE-Test-Publisher'); tx.add_tag('SHA-256',sha)
+                    tx.sign(); tx.send(); txid=tx.id; summary['uploaded']+=1
+                    state['files'][sha]={'id':txid,'file':str(fp),'uploaded_at':int(time.time())}; atomic(statep,state)
+                    print(f"[{ch['name']}:{n}] uploaded {fp.name} -> {txid}")
+                refs.append({'arweave_id':txid,'url':f'{gateway}/{txid}','sha256':sha})
+            text=item['text'].rstrip()
+            if refs: text += '\n\n' + '\n'.join(x['url'] for x in refs)
+            prepared.append({'id':item.get('id',str(n)),'text':text,'images':refs})
+            summary['posts']+=1
+        atomic(outp,prepared)
+        print(f"Ready {ch['name']}: {outp} ({len(prepared)} posts)")
+    print(json.dumps(summary,ensure_ascii=False,indent=2))
+if __name__=='__main__': main()