#!/usr/bin/env python3 import argparse, asyncio, base64, hashlib, json, os, random, struct, sys, time, uuid from pathlib import Path from datetime import datetime, timedelta from zoneinfo import ZoneInfo from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from websockets.asyncio.client import connect ZERO32 = bytes(32) FRAME_CODE_V1=1; TEXT_TYPE=1; TEXT_POST=10; VERSION=1 def atomic_json(path, obj): path=Path(path); path.parent.mkdir(parents=True, exist_ok=True) tmp=path.with_suffix(path.suffix+'.tmp') tmp.write_text(json.dumps(obj,ensure_ascii=False,indent=2),encoding='utf-8') os.replace(tmp,path) def load_json(path, default=None): p=Path(path) return json.loads(p.read_text(encoding='utf-8')) if p.exists() else default def avro_long(n): n=(n<<1) ^ (n>>63); out=bytearray() while n & ~0x7f: out.append((n&0x7f)|0x80); n >>= 7 out.append(n); return bytes(out) def avro_tags(tags): out=bytearray() if tags: out += avro_long(len(tags)) for k,v in tags: kb=k.encode(); vb=v.encode(); out+=avro_long(len(kb))+kb+avro_long(len(vb))+vb out += avro_long(0); return bytes(out) def deep_hash(x): H=lambda b: hashlib.sha384(b).digest() if isinstance(x,(bytes,bytearray)): b=bytes(x); return H(H(f'blob{len(b)}'.encode())+H(b)) acc=H(f'list{len(x)}'.encode()) for child in x: acc=H(acc+deep_hash(child)) return acc def signing_message(owner,tags,data): raw=avro_tags(tags) return deep_hash([b'dataitem',b'1',b'2',owner,b'',b'',raw,data]) def data_item(priv64,tags,data): if len(priv64) not in (32,64): raise ValueError('Solana key JSON must contain 32 or 64 bytes') seed=priv64[:32]; owner=priv64[32:64] if len(priv64)==64 else Ed25519PrivateKey.from_private_bytes(seed).public_key().public_bytes_raw() sig=Ed25519PrivateKey.from_private_bytes(seed).sign(signing_message(owner,tags,data)) rawtags=avro_tags(tags) return struct.pack('65535: raise ValueError('post text too long') return struct.pack('>ii32siH',line_code,prev_line_num,prev_line_hash,this_line_num,len(tb))+tb def frame(prev_hash, block_num, body, ts=None): ts=int(time.time()) if ts is None else int(ts); size=56+len(body) return struct.pack('>H32siiqHHH',FRAME_CODE_V1,prev_hash,size,block_num,ts,TEXT_TYPE,TEXT_POST,VERSION)+body def h32(b): return hashlib.sha256(b).digest() def hx(b): return b.hex() async def ws_call(url, op, payload, timeout=20): req={'op':op,'requestId':str(uuid.uuid4()),'payload':payload} async with connect(url, open_timeout=timeout, close_timeout=5) as ws: await ws.send(json.dumps(req,separators=(',',':'))) end=time.monotonic()+timeout while True: left=end-time.monotonic() if left<=0: raise TimeoutError(op) msg=await asyncio.wait_for(ws.recv(),left) obj=json.loads(msg) if obj.get('requestId')==req['requestId']: return obj async def head(cfg): r=await ws_call(cfg['server_ws'],'ListBlockchainHeads',{}) if r.get('status')!=200: raise RuntimeError(f'ListBlockchainHeads: {r}') for x in r.get('payload',{}).get('blockchains',[]): if x.get('blockchainName')==cfg['blockchain_name']: return int(x.get('lastBlockNumber',-1)), x.get('lastBlockHash','') raise RuntimeError('blockchain not found on server') async def block(cfg,n): r=await ws_call(cfg['server_ws'],'GetBlockchainBlock',{'blockchainName':cfg['blockchain_name'],'blockNumber':n}) if r.get('status')!=200: raise RuntimeError(f'GetBlockchainBlock({n}): {r}') p=r.get('payload',r) return p.get('blockHash') or r.get('blockHash'), p.get('blockBytesB64') or r.get('blockBytesB64') async def channel_tail(cfg, root_hash): r=await ws_call(cfg['server_ws'],'GetChannelMessages',{'channel':{'ownerBlockchainName':cfg['blockchain_name'],'channelRootBlockNumber':cfg['channel_root_block_number'],'channelRootBlockHash':root_hash},'limit':1,'sort':'desc'}) if r.get('status')!=200: raise RuntimeError(f'GetChannelMessages: {r}') items=r.get('payload',{}).get('messages',[]) if not items: return int(cfg['channel_root_block_number']), bytes.fromhex(root_hash), -1 m=items[0]; ref=m.get('messageRef') or {} return int(ref.get('blockNumber')), bytes.fromhex(ref.get('blockHash','00'*32)), int(m.get('lineStep') or 0) def read_key(path): a=load_json(path) if not isinstance(a,list): raise ValueError('key file must be Solana JSON byte array') return bytes(int(x)&255 for x in a) def queue_items(cfg): q=load_json(cfg['queue_file']) if not isinstance(q,list) or not q: raise ValueError('queue must be non-empty JSON array') for x in q: if not isinstance(x,dict) or not str(x.get('text','')).strip(): raise ValueError('each queue item needs text') return q def delay(cfg): return random.randint(int(cfg['min_interval_seconds']),int(cfg['max_interval_seconds'])) async def init_state(cfg): bn,bhash=await head(cfg) root_hash,_=await block(cfg,int(cfg['channel_root_block_number'])) line_num,line_hash=await channel_tail(cfg,root_hash) st={'version':1,'next_index':0,'last_block_number':bn,'last_block_hash':bhash,'last_line_number':line_num,'last_line_hash':hx(line_hash),'next_publish_at':int(time.time())+delay(cfg),'pending':None,'published_total':0} atomic_json(cfg['state_file'],st); return st async def reconcile(cfg,st): p=st.get('pending') bn,bhash=await head(cfg) if p: if bn==p['block_number'] and bhash.lower()==p['block_hash'].lower(): st['last_block_number']=bn; st['last_block_hash']=bhash; st['last_line_number']=p['this_line_number']; st['last_line_hash']=p['block_hash']; st['next_index']=p['next_index_after']; st['published_total']=int(st.get('published_total',0))+1; st['pending']=None; st['next_publish_at']=int(time.time())+delay(cfg); atomic_json(cfg['state_file'],st); print('Recovered accepted pending block',bn) elif bn==p['previous_block_number'] and bhash.lower()==p['previous_block_hash'].lower(): print('Pending block was not accepted; it will be retried') else: raise RuntimeError(f'chain changed while pending: server={bn}:{bhash} state={p}') else: if bn!=st['last_block_number'] or bhash.lower()!=str(st['last_block_hash']).lower(): raise RuntimeError('server chain head differs from publisher state; refusing to fork') return st async def publish_one(cfg,st,items): if st.get('pending'): p=st['pending']; raw=base64.b64decode(p['data_item_b64']); block_num=p['block_number']; prev=p['previous_block_hash'] else: idx=int(st['next_index']) if idx>=len(items): if cfg.get('loop_queue',False): idx=0 else: print('Queue exhausted. Nothing to publish.'); return False text=items[idx]['text'].strip(); block_num=int(st['last_block_number'])+1; prev_hash=bytes.fromhex(st['last_block_hash']); this_line=int(st['last_line_number'])+1 body=post_body(int(cfg['channel_root_block_number']),int(st['last_line_number']),bytes.fromhex(st['last_line_hash']),this_line,text) fr=frame(prev_hash,block_num,body); block_hash=h32(fr); tags=[('App','test5590'),('c',cfg['channel_name'].strip().lower())] raw=data_item(read_key(cfg['key_file']),tags,fr) nextidx=idx+1 p={'queue_index':idx,'next_index_after':nextidx,'block_number':block_num,'block_hash':hx(block_hash),'previous_block_number':st['last_block_number'],'previous_block_hash':st['last_block_hash'],'this_line_number':this_line,'data_item_b64':base64.b64encode(raw).decode()} st['pending']=p; atomic_json(cfg['state_file'],st); prev=st['last_block_hash'] r=await ws_call(cfg['server_ws'],'AddBlock',{'blockchainName':cfg['blockchain_name'],'blockNumber':block_num,'prevBlockHash':prev,'blockBytesB64':base64.b64encode(raw).decode()},timeout=30) if r.get('status')!=200: raise RuntimeError(f'AddBlock failed: {r}') server_hash=(r.get('payload') or {}).get('serverLastBlockHash') or (r.get('payload') or {}).get('serverLastGlobalHash') if server_hash and server_hash.lower()!=p['block_hash'].lower(): raise RuntimeError('server accepted different hash') st['last_block_number']=block_num; st['last_block_hash']=p['block_hash']; st['last_line_number']=p['this_line_number']; st['last_line_hash']=p['block_hash']; st['next_index']=p['next_index_after']; st['published_total']=int(st.get('published_total',0))+1; st['pending']=None; st['next_publish_at']=int(time.time())+delay(cfg); atomic_json(cfg['state_file'],st) print(f"Published queue[{p['queue_index']}] as block #{block_num}; next at {time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(st['next_publish_at']))}") return True async def init_multi_state(cfg): bn,bhash=await head(cfg) chans={} now=int(time.time()) for ch in cfg['channels']: local=dict(cfg); local.update(ch) root_hash,_=await block(local,int(ch['channel_root_block_number'])) line_block,line_hash,line_step=await channel_tail(local,root_hash) chans[ch['channel_name']]={'next_index':0,'last_line_block_number':line_block, 'last_line_number':line_step,'last_line_hash':hx(line_hash), 'next_publish_at':next_channel_time(cfg,ch,now), 'published_total':0} st={'version':2,'last_block_number':bn,'last_block_hash':bhash,'pending':None,'channels':chans} atomic_json(cfg['state_file'],st); return st def channel_delay(cfg,ch): return random.randint(int(ch.get('min_interval_seconds',cfg['min_interval_seconds'])),int(ch.get('max_interval_seconds',cfg['max_interval_seconds']))) def _hm(v): h,m=str(v).split(':',1); return int(h),int(m) def _quiet(dt,ch,cfg): q=ch.get('quiet_hours',cfg.get('quiet_hours',{})) if not q or not q.get('enabled',True): return False sh,sm=_hm(q.get('start','22:00')); eh,em=_hm(q.get('end','08:00')) t=dt.hour*60+dt.minute; a=sh*60+sm; b=eh*60+em return (a<=t or tb else (a<=t0: q=ch.get('quiet_hours',cfg.get('quiet_hours',{})) if not q or not q.get('enabled',True): return int((dt+timedelta(seconds=remain)).timestamp()) sh,sm=_hm(q.get('start','22:00')); eh,em=_hm(q.get('end','08:00')) if _quiet(dt,ch,cfg): end=dt.replace(hour=eh,minute=em,second=0,microsecond=0) if end<=dt: end+=timedelta(days=1) dt=end; continue qs=dt.replace(hour=sh,minute=sm,second=0,microsecond=0) if qs<=dt: qs+=timedelta(days=1) usable=max(0,int((qs-dt).total_seconds())) if remain<=usable: return int((dt+timedelta(seconds=remain)).timestamp()) remain-=usable; dt=qs return int(dt.timestamp()) def next_channel_time(cfg,ch,start_ts=None): return active_deadline(cfg,ch,channel_delay(cfg,ch),start_ts) async def reconcile_multi(cfg,st): bn,bhash=await head(cfg); p=st.get('pending') if p: cs=st['channels'][p['channel_name']] if bn==p['block_number'] and bhash.lower()==p['block_hash'].lower(): st['last_block_number']=bn; st['last_block_hash']=bhash cs['last_line_block_number']=p['block_number']; cs['last_line_number']=p['this_line_number']; cs['last_line_hash']=p['block_hash'] cs['next_index']=p['next_index_after']; cs['published_total']=int(cs.get('published_total',0))+1 ch=next(x for x in cfg['channels'] if x['channel_name']==p['channel_name']) cs['next_publish_at']=next_channel_time(cfg,ch); st['pending']=None; atomic_json(cfg['state_file'],st) elif bn!=p['previous_block_number'] or bhash.lower()!=p['previous_block_hash'].lower(): raise RuntimeError('global chain changed while a pending block exists; refusing to fork') elif bn!=st['last_block_number'] or bhash.lower()!=str(st['last_block_hash']).lower(): raise RuntimeError('server global chain head differs from publisher state; refusing to fork') return st async def publish_channel(cfg,st,ch,force=False): cs=st['channels'][ch['channel_name']]; items=queue_items(ch) idx=int(cs['next_index']) if idx>=len(items): if ch.get('loop_queue',cfg.get('loop_queue',False)): idx=0 else: cs['next_publish_at']=int(time.time())+3600; atomic_json(cfg['state_file'],st); return False text=items[idx]['text'].strip(); block_num=int(st['last_block_number'])+1 prev_hash=bytes.fromhex(st['last_block_hash']); this_line=int(cs['last_line_number'])+1 prev_line_block=int(cs.get('last_line_block_number', ch['channel_root_block_number'])) body=post_body(int(ch['channel_root_block_number']),prev_line_block,bytes.fromhex(cs['last_line_hash']),this_line,text) fr=frame(prev_hash,block_num,body); block_hash=h32(fr) tags=[('App','test5590'),('c',ch['channel_name'].strip().lower())] raw=data_item(read_key(cfg['key_file']),tags,fr) p={'channel_name':ch['channel_name'],'queue_index':idx,'next_index_after':idx+1,'block_number':block_num, 'block_hash':hx(block_hash),'previous_block_number':st['last_block_number'],'previous_block_hash':st['last_block_hash'], 'this_line_number':this_line,'data_item_b64':base64.b64encode(raw).decode()} st['pending']=p; atomic_json(cfg['state_file'],st) r=await ws_call(cfg['server_ws'],'AddBlock',{'blockchainName':cfg['blockchain_name'],'blockNumber':block_num, 'prevBlockHash':st['last_block_hash'],'blockBytesB64':base64.b64encode(raw).decode()},timeout=30) if r.get('status')!=200: raise RuntimeError(f'AddBlock failed: {r}') st['last_block_number']=block_num; st['last_block_hash']=p['block_hash'] cs['last_line_block_number']=block_num; cs['last_line_number']=this_line; cs['last_line_hash']=p['block_hash']; cs['next_index']=idx+1 cs['published_total']=int(cs.get('published_total',0))+1; cs['next_publish_at']=next_channel_time(cfg,ch) st['pending']=None; atomic_json(cfg['state_file'],st) print(f"Published {ch['channel_name']}[{idx}] as global block #{block_num}") return True async def main_async(args): cfg=load_json(args.config); base=Path(args.config).resolve().parent for k in ('key_file','state_file'): p=Path(cfg[k]); cfg[k]=str(p if p.is_absolute() else base/p) if not isinstance(cfg.get('channels'),list) or not cfg['channels']: raise ValueError('channels[] required') for ch in cfg['channels']: p=Path(ch['queue_file']); ch['queue_file']=str(p if p.is_absolute() else base/p) for k in ('channel_name','channel_root_block_number','queue_file'): if k not in ch: raise ValueError('channel missing '+k) st=load_json(cfg['state_file']) if st is None: st=await init_multi_state(cfg) st=await reconcile_multi(cfg,st) if args.command=='status': print(json.dumps(st,ensure_ascii=False,indent=2)); return if args.command=='publish-now': name=args.channel or cfg['channels'][0]['channel_name'] ch=next((x for x in cfg['channels'] if x['channel_name']==name),None) if not ch: raise ValueError('unknown channel '+name) await publish_channel(cfg,st,ch,True); return while True: st=await reconcile_multi(cfg,load_json(cfg['state_file'])) now=int(time.time()) due=sorted(cfg['channels'],key=lambda c:st['channels'][c['channel_name']]['next_publish_at']) ch=due[0]; wait=max(0,int(st['channels'][ch['channel_name']]['next_publish_at'])-now) if wait: await asyncio.sleep(wait) await publish_channel(cfg,st,ch) def main(): ap=argparse.ArgumentParser(description='SHiNE multi-channel test publisher') ap.add_argument('command',choices=['run','status','publish-now'],nargs='?',default='run') ap.add_argument('--config',default='config.json'); ap.add_argument('--channel',default='') a=ap.parse_args() try: asyncio.run(main_async(a)) except KeyboardInterrupt: pass except Exception as e: print('ERROR:',e,file=sys.stderr); sys.exit(1) if __name__=='__main__': main()