Ускорить публикацию блоков в Arweave

This commit is contained in:
AidarKC
2026-10-03 16:39:35 +03:00
parent aca1e3054e
commit daa221a158
13 changed files with 629 additions and 240 deletions
@@ -2,17 +2,32 @@ package server.archive;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import server.sync.ArweavePublishSignal;
import java.util.Base64;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/** Fixed-delay publisher. Empty cycles are no-ops. */
/**
* Publisher с двумя путями доставки:
* 1) Turbo — немедленная неблокирующая попытка для только что принятого блока;
* 2) плановый fixed-delay цикл — страховочная добивка только зависших pending-записей.
*/
public final class ArweaveBlockPublisherScheduler {
private static final Logger log = LoggerFactory.getLogger(ArweaveBlockPublisherScheduler.class);
private static final long RETRY_MIN_PENDING_AGE_MS = 60_000L;
private static final AtomicBoolean STARTED = new AtomicBoolean(false);
private static final AtomicBoolean IMMEDIATE_DRAIN_SCHEDULED = new AtomicBoolean(false);
private static final ConcurrentLinkedQueue<ArweavePublishSignal.PendingRef> IMMEDIATE_QUEUE = new ConcurrentLinkedQueue<>();
private static final ConcurrentHashMap<String, Long> PENDING_SINCE_MS = new ConcurrentHashMap<>();
private static ScheduledExecutorService executor;
private static ArweaveBlockPublisherService service;
private static ArweaveBlocksConfig config;
private ArweaveBlockPublisherScheduler() {}
@@ -23,15 +38,105 @@ public final class ArweaveBlockPublisherScheduler {
if (!cfg.publishEnabled()) { log.info("Arweave user-block publisher disabled (mode=none)"); return; }
if (!STARTED.compareAndSet(false,true)) return;
try {
ArweaveBlockPublisherService service = new ArweaveBlockPublisherService(cfg);
config = cfg;
service = new ArweaveBlockPublisherService(cfg);
executor = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "shine-arweave-block-publisher"); t.setDaemon(true); return t;
});
Runnable task = () -> { try { service.runCycle(); } catch (Exception e) { log.error("Arweave block publish cycle failed", e); } };
executor.scheduleWithFixedDelay(task, 0, cfg.publishIntervalMinutes(), TimeUnit.MINUTES);
log.info("Arweave user-block publisher enabled: mode={} interval={}m", cfg.publishMode(), cfg.publishIntervalMinutes());
} catch (Exception e) { STARTED.set(false); log.error("Arweave user-block publisher failed to start", e); }
if (cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO) {
ArweavePublishSignal.setListener(ArweaveBlockPublisherScheduler::enqueueImmediateTurboPublish);
}
Runnable retryTask = () -> {
try {
if (config.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO) {
long nowMs = System.currentTimeMillis();
service.runRetryCycle(
id -> isOldEnoughForRetry(id, nowMs),
id -> PENDING_SINCE_MS.remove(idKey(id))
);
} else {
service.runCycle();
}
} catch (Exception e) {
log.error("Arweave block publish retry cycle failed", e);
}
};
long retryInitialDelayMinutes = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO
? cfg.publishIntervalMinutes()
: 0L;
executor.scheduleWithFixedDelay(retryTask, retryInitialDelayMinutes, cfg.publishIntervalMinutes(), TimeUnit.MINUTES);
log.info("Arweave user-block publisher enabled: mode={} immediateTurbo={} retryInterval={}m retryMinAge={}s",
cfg.publishMode(), cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO,
cfg.publishIntervalMinutes(), RETRY_MIN_PENDING_AGE_MS / 1000L);
} catch (Exception e) {
ArweavePublishSignal.clearListener();
STARTED.set(false);
log.error("Arweave user-block publisher failed to start", e);
}
}
public static void close() { ScheduledExecutorService e=executor; if(e!=null)e.shutdownNow(); }
private static void enqueueImmediateTurboPublish(ArweavePublishSignal.PendingRef item) {
if (!STARTED.get() || config == null || config.publishMode() != ArweaveBlocksConfig.PublishMode.TURBO) return;
PENDING_SINCE_MS.putIfAbsent(idKey(item.dataItemId()), System.currentTimeMillis());
IMMEDIATE_QUEUE.offer(item);
scheduleImmediateDrain();
}
private static boolean isOldEnoughForRetry(byte[] dataItemId, long nowMs) {
Long pendingSince = PENDING_SINCE_MS.get(idKey(dataItemId));
// Нет записи в памяти => pending пережил старт процесса и уже считается зависшим.
return pendingSince == null || nowMs - pendingSince >= RETRY_MIN_PENDING_AGE_MS;
}
private static String idKey(byte[] dataItemId) {
return Base64.getUrlEncoder().withoutPadding().encodeToString(dataItemId);
}
private static void scheduleImmediateDrain() {
ScheduledExecutorService currentExecutor = executor;
if (currentExecutor == null || currentExecutor.isShutdown()) return;
if (!IMMEDIATE_DRAIN_SCHEDULED.compareAndSet(false, true)) return;
try {
currentExecutor.execute(ArweaveBlockPublisherScheduler::drainImmediateQueue);
} catch (RuntimeException e) {
IMMEDIATE_DRAIN_SCHEDULED.set(false);
throw e;
}
}
private static void drainImmediateQueue() {
try {
ArweavePublishSignal.PendingRef item;
while ((item = IMMEDIATE_QUEUE.poll()) != null) {
try {
boolean resolved = service.publishAcceptedNow(item);
if (resolved) PENDING_SINCE_MS.remove(idKey(item.dataItemId()));
} catch (Exception e) {
// Turbo недоступен: не держим поток клиента и не молотим всю очередь одной и той же ошибкой.
// Все элементы уже надёжно лежат в БД с pending=TRUE; плановый цикл подберёт их после min-age.
log.warn("Immediate Turbo publish failed; pending blocks will be retried by the periodic cycle: {}", e.getMessage());
IMMEDIATE_QUEUE.clear();
break;
}
}
} finally {
IMMEDIATE_DRAIN_SCHEDULED.set(false);
if (!IMMEDIATE_QUEUE.isEmpty()) scheduleImmediateDrain();
}
}
public static void close() {
ArweavePublishSignal.clearListener();
IMMEDIATE_QUEUE.clear();
PENDING_SINCE_MS.clear();
IMMEDIATE_DRAIN_SCHEDULED.set(false);
ScheduledExecutorService e = executor;
executor = null;
service = null;
config = null;
STARTED.set(false);
if (e != null) e.shutdownNow();
}
}
@@ -6,9 +6,12 @@ import shine.db.dao.BlocksDAO;
import shine.db.dao.KeyRotationCandidateBlocksDAO;
import shine.db.entities.BlockEntry;
import shine.db.entities.KeyRotationCandidateBlockEntry;
import server.sync.ArweavePublishSignal;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Predicate;
/** Publishes locally-created signed user DataItems through the configured transport. */
public final class ArweaveBlockPublisherService {
@@ -26,7 +29,47 @@ public final class ArweaveBlockPublisherService {
this.turboUploader = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO ? new TurboDataItemUploader(cfg) : null;
}
/**
* Мгновенная попытка для только что принятого DataItem. Вызывается только из фонового executor-а:
* сетевой запрос AddBlock не ждёт результат Turbo. Если запись уже успел обработать другой цикл, это no-op.
* При ошибке pending-флаг в БД не меняется, поэтому обычный runCycle() повторит публикацию позже.
*/
public boolean publishAcceptedNow(ArweavePublishSignal.PendingRef pending) throws Exception {
if (cfg.publishMode() != ArweaveBlocksConfig.PublishMode.TURBO) return false;
byte[] id = pending.dataItemId();
if (pending.kind() == ArweavePublishSignal.Kind.KEY_ROTATION_CANDIDATE) {
KeyRotationCandidateBlockEntry entry = candidateBlocksDAO.getPendingArweaveByDataItemId(id);
if (entry == null) return true;
byte[] raw = entry.getBlockBytes();
if (raw == null || raw.length == 0) return false;
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
candidateBlocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
log.debug("Turbo immediately published key-rotation DataItem {} chain={} block={}",
result.dataItemId(), entry.getCandidateBlockchainName(), entry.getBlockNumber());
return true;
}
BlockEntry entry = blocksDAO.getPendingArweaveByDataItemId(id);
if (entry == null) return true;
byte[] raw = entry.getBlockBytes();
if (raw == null || raw.length == 0) return false;
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
blocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
log.debug("Turbo immediately published SHiNE DataItem {} chain={} block={}",
result.dataItemId(), entry.getBchName(), entry.getBlockNumber());
return true;
}
public int runCycle() throws Exception {
return runCycle(id -> true, id -> {});
}
/** Плановый retry может отфильтровать слишком свежие pending-записи и получить callback успешных id. */
public int runRetryCycle(Predicate<byte[]> eligible, Consumer<byte[]> onPublished) throws Exception {
return runCycle(eligible == null ? id -> true : eligible, onPublished == null ? id -> {} : onPublished);
}
private int runCycle(Predicate<byte[]> eligible, Consumer<byte[]> onPublished) throws Exception {
if (cfg.publishMode() == ArweaveBlocksConfig.PublishMode.NONE) return 0;
int maxItems = cfg.publishMaxItems();
@@ -35,10 +78,11 @@ public final class ArweaveBlockPublisherService {
// Ротация приоритетна: пользователь уже заблокирован для обычной записи и ждёт завершения копирования.
List<KeyRotationCandidateBlockEntry> rotationCandidates = candidateBlocksDAO.listPendingArweave(maxItems);
rotationCandidates = rotationCandidates.stream().filter(e -> eligible.test(e.getDataItemId())).toList();
if (!rotationCandidates.isEmpty()) {
attempted += rotationCandidates.size();
published += switch (cfg.publishMode()) {
case TURBO -> publishTurboRotation(rotationCandidates);
case TURBO -> publishTurboRotation(rotationCandidates, onPublished);
case ARWEAVE -> publishDirectArweaveRotation(rotationCandidates);
case NONE -> 0;
};
@@ -48,16 +92,17 @@ public final class ArweaveBlockPublisherService {
if (remaining == 0) return published;
List<BlockEntry> candidates = blocksDAO.listPendingArweave(remaining);
candidates = candidates.stream().filter(e -> eligible.test(e.getDataItemId())).toList();
if (candidates.isEmpty()) return published;
published += switch (cfg.publishMode()) {
case TURBO -> publishTurbo(candidates);
case TURBO -> publishTurbo(candidates, onPublished);
case ARWEAVE -> publishDirectArweave(candidates);
case NONE -> 0;
};
return published;
}
private int publishTurboRotation(List<KeyRotationCandidateBlockEntry> candidates) throws Exception {
private int publishTurboRotation(List<KeyRotationCandidateBlockEntry> candidates, Consumer<byte[]> onPublished) throws Exception {
int published = 0;
Exception firstFailure = null;
for (KeyRotationCandidateBlockEntry e : candidates) {
@@ -67,6 +112,7 @@ public final class ArweaveBlockPublisherService {
try {
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
candidateBlocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
onPublished.accept(id);
published++;
log.debug("Turbo published key-rotation DataItem {} chain={} block={}",
result.dataItemId(), e.getCandidateBlockchainName(), e.getBlockNumber());
@@ -81,7 +127,7 @@ public final class ArweaveBlockPublisherService {
return published;
}
private int publishTurbo(List<BlockEntry> candidates) throws Exception {
private int publishTurbo(List<BlockEntry> candidates, Consumer<byte[]> onPublished) throws Exception {
int published = 0;
Exception firstFailure = null;
for (BlockEntry e : candidates) {
@@ -91,6 +137,7 @@ public final class ArweaveBlockPublisherService {
try {
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
blocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
onPublished.accept(id);
published++;
log.debug("Turbo published SHiNE DataItem {} chain={} block={}", result.dataItemId(), e.getBchName(), e.getBlockNumber());
} catch (Exception ex) {
@@ -68,6 +68,16 @@ public final class BlocksDAO {
}
}
/** Возвращает ещё не опубликованный пользовательский DataItem для быстрой Turbo-попытки. */
public BlockEntry getPendingArweaveByDataItemId(byte[] dataItemId) throws SQLException {
if (dataItemId == null || dataItemId.length != 32) return null;
String sql = baseSelect() + " WHERE data_item_id=? AND arweave_publish_pending=TRUE LIMIT 1";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setBytes(1, dataItemId);
try (ResultSet rs = ps.executeQuery()) { return rs.next() ? mapRow(rs) : null; }
}
}
public List<BlockEntry> listPendingArweave(int limit) throws SQLException {
int safeLimit = Math.max(1, Math.min(limit, 100_000));
String sql = """
@@ -132,6 +132,20 @@ public final class KeyRotationCandidateBlocksDAO {
}
}
/** Возвращает ещё не опубликованный candidate DataItem для быстрой Turbo-попытки. */
public KeyRotationCandidateBlockEntry getPendingArweaveByDataItemId(byte[] dataItemId) throws SQLException {
if (dataItemId == null || dataItemId.length != 32) return null;
try (Connection c = db.getConnection();
PreparedStatement ps = c.prepareStatement("""
SELECT * FROM key_rotation_candidate_blocks
WHERE data_item_id = ? AND arweave_publish_pending = TRUE
LIMIT 1
""")) {
ps.setBytes(1, dataItemId);
try (ResultSet rs = ps.executeQuery()) { return rs.next() ? map(rs) : null; }
}
}
/** Candidate-блоки имеют приоритет перед обычной очередью publisher-а. */
public List<KeyRotationCandidateBlockEntry> listPendingArweave(int limit) throws SQLException {
if (limit <= 0) return List.of();
@@ -32,6 +32,7 @@ import server.logic.ws_protocol.JSON.handlers.channels.ChannelNamesStateBootstra
import server.logic.ws_protocol.JSON.handlers.channels.UserCountersSupport;
import server.logic.ws_protocol.WireCodes;
import server.sync.AddBlockSyncService;
import server.sync.ArweavePublishSignal;
import shine.db.channels.ChannelNameRules;
import shine.db.dao.BlockchainStateDAO;
import shine.db.dao.BlocksDAO;
@@ -720,6 +721,11 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
upsertChat200StateFromCreate(chat200CreateSeed);
}
maybeApplyChat200Command(blockchainName, block);
if (publishToArweave) {
// Только ставим быструю публикацию в очередь: ответ клиенту не ждёт сеть Turbo/Arweave.
ArweavePublishSignal.userBlockAccepted(be.getDataItemId());
}
} catch (Exception e) {
if (isChannelSlugConflict(e)) {
return new AddBlockResult(409, "channel_name_already_exists", serverLastNum, serverLastHashHex);
@@ -13,6 +13,7 @@ import server.logic.ws_protocol.JSON.handlers.JsonMessageHandler;
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler_utils.BlockchainLocks;
import server.logic.ws_protocol.JSON.handlers.keyRotation.entyties.Net_KeyRotationAddBlock_Request;
import server.logic.ws_protocol.JSON.utils.NetExceptionResponseFactory;
import server.sync.ArweavePublishSignal;
import shine.db.DbController;
import shine.db.dao.BlocksDAO;
import shine.db.dao.KeyRotationCandidateBlocksDAO;
@@ -164,6 +165,10 @@ public final class Net_KeyRotationAddBlock_Handler implements JsonMessageHandler
c.setAutoCommit(oldAutoCommit);
}
// Как и обычный пользовательский блок, candidate сразу ставится в неблокирующую Turbo-очередь.
// При сетевой ошибке pending-флаг остаётся TRUE и его подберёт плановый retry.
ArweavePublishSignal.keyRotationCandidateAccepted(entry.getDataItemId());
// progressCurrent обновляет Arweave publisher только после реальной публикации.
KeyRotationSessionEntry refreshed = rotations.getActiveByLogin(login);
return KeyRotationApiSupport.response(req.getOp(), req.getRequestId(), refreshed == null ? session : refreshed);
@@ -0,0 +1,74 @@
package server.sync;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
/**
* Неблокирующий мост между приёмом блока по протоколу и Arweave/Turbo publisher-ом.
*
* Net-protocol не зависит от archive-модуля, поэтому publisher регистрирует здесь лёгкий listener.
* Listener обязан только поставить ссылку на уже сохранённый DataItem в свою очередь и не выполнять
* сетевую публикацию в потоке клиента.
*/
public final class ArweavePublishSignal {
private static final Logger log = LoggerFactory.getLogger(ArweavePublishSignal.class);
public enum Kind {
USER_BLOCK,
KEY_ROTATION_CANDIDATE
}
public record PendingRef(Kind kind, byte[] dataItemId) {
public PendingRef {
if (kind == null) throw new IllegalArgumentException("kind is null");
if (dataItemId == null || dataItemId.length != 32) {
throw new IllegalArgumentException("dataItemId must be 32 bytes");
}
dataItemId = Arrays.copyOf(dataItemId, dataItemId.length);
}
@Override
public byte[] dataItemId() {
return Arrays.copyOf(dataItemId, dataItemId.length);
}
}
@FunctionalInterface
public interface Listener {
void onPending(PendingRef item);
}
private static volatile Listener listener;
private ArweavePublishSignal() {}
public static void setListener(Listener value) {
listener = value;
}
public static void clearListener() {
listener = null;
}
public static void userBlockAccepted(byte[] dataItemId) {
publish(new PendingRef(Kind.USER_BLOCK, dataItemId));
}
public static void keyRotationCandidateAccepted(byte[] dataItemId) {
publish(new PendingRef(Kind.KEY_ROTATION_CANDIDATE, dataItemId));
}
private static void publish(PendingRef item) {
Listener current = listener;
if (current == null) return;
try {
current.onPending(item);
} catch (RuntimeException e) {
// Ошибка постановки в быструю очередь не должна превращать уже принятый блок в ошибку клиента:
// arweave_publish_pending остаётся TRUE, и плановый retry подберёт его позже.
log.warn("Cannot enqueue immediate Arweave/Turbo publish; periodic retry will keep the block pending", e);
}
}
}