SHA256
Ускорить публикацию блоков в Arweave
This commit is contained in:
+112
-7
@@ -2,17 +2,32 @@ package server.archive;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import server.sync.ArweavePublishSignal;
|
||||
|
||||
import java.util.Base64;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/** Fixed-delay publisher. Empty cycles are no-ops. */
|
||||
/**
|
||||
* Publisher с двумя путями доставки:
|
||||
* 1) Turbo — немедленная неблокирующая попытка для только что принятого блока;
|
||||
* 2) плановый fixed-delay цикл — страховочная добивка только зависших pending-записей.
|
||||
*/
|
||||
public final class ArweaveBlockPublisherScheduler {
|
||||
private static final Logger log = LoggerFactory.getLogger(ArweaveBlockPublisherScheduler.class);
|
||||
private static final long RETRY_MIN_PENDING_AGE_MS = 60_000L;
|
||||
private static final AtomicBoolean STARTED = new AtomicBoolean(false);
|
||||
private static final AtomicBoolean IMMEDIATE_DRAIN_SCHEDULED = new AtomicBoolean(false);
|
||||
private static final ConcurrentLinkedQueue<ArweavePublishSignal.PendingRef> IMMEDIATE_QUEUE = new ConcurrentLinkedQueue<>();
|
||||
private static final ConcurrentHashMap<String, Long> PENDING_SINCE_MS = new ConcurrentHashMap<>();
|
||||
|
||||
private static ScheduledExecutorService executor;
|
||||
private static ArweaveBlockPublisherService service;
|
||||
private static ArweaveBlocksConfig config;
|
||||
|
||||
private ArweaveBlockPublisherScheduler() {}
|
||||
|
||||
@@ -23,15 +38,105 @@ public final class ArweaveBlockPublisherScheduler {
|
||||
if (!cfg.publishEnabled()) { log.info("Arweave user-block publisher disabled (mode=none)"); return; }
|
||||
if (!STARTED.compareAndSet(false,true)) return;
|
||||
try {
|
||||
ArweaveBlockPublisherService service = new ArweaveBlockPublisherService(cfg);
|
||||
config = cfg;
|
||||
service = new ArweaveBlockPublisherService(cfg);
|
||||
executor = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "shine-arweave-block-publisher"); t.setDaemon(true); return t;
|
||||
});
|
||||
Runnable task = () -> { try { service.runCycle(); } catch (Exception e) { log.error("Arweave block publish cycle failed", e); } };
|
||||
executor.scheduleWithFixedDelay(task, 0, cfg.publishIntervalMinutes(), TimeUnit.MINUTES);
|
||||
log.info("Arweave user-block publisher enabled: mode={} interval={}m", cfg.publishMode(), cfg.publishIntervalMinutes());
|
||||
} catch (Exception e) { STARTED.set(false); log.error("Arweave user-block publisher failed to start", e); }
|
||||
|
||||
if (cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO) {
|
||||
ArweavePublishSignal.setListener(ArweaveBlockPublisherScheduler::enqueueImmediateTurboPublish);
|
||||
}
|
||||
|
||||
Runnable retryTask = () -> {
|
||||
try {
|
||||
if (config.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO) {
|
||||
long nowMs = System.currentTimeMillis();
|
||||
service.runRetryCycle(
|
||||
id -> isOldEnoughForRetry(id, nowMs),
|
||||
id -> PENDING_SINCE_MS.remove(idKey(id))
|
||||
);
|
||||
} else {
|
||||
service.runCycle();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Arweave block publish retry cycle failed", e);
|
||||
}
|
||||
};
|
||||
long retryInitialDelayMinutes = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO
|
||||
? cfg.publishIntervalMinutes()
|
||||
: 0L;
|
||||
executor.scheduleWithFixedDelay(retryTask, retryInitialDelayMinutes, cfg.publishIntervalMinutes(), TimeUnit.MINUTES);
|
||||
log.info("Arweave user-block publisher enabled: mode={} immediateTurbo={} retryInterval={}m retryMinAge={}s",
|
||||
cfg.publishMode(), cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO,
|
||||
cfg.publishIntervalMinutes(), RETRY_MIN_PENDING_AGE_MS / 1000L);
|
||||
} catch (Exception e) {
|
||||
ArweavePublishSignal.clearListener();
|
||||
STARTED.set(false);
|
||||
log.error("Arweave user-block publisher failed to start", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static void close() { ScheduledExecutorService e=executor; if(e!=null)e.shutdownNow(); }
|
||||
private static void enqueueImmediateTurboPublish(ArweavePublishSignal.PendingRef item) {
|
||||
if (!STARTED.get() || config == null || config.publishMode() != ArweaveBlocksConfig.PublishMode.TURBO) return;
|
||||
PENDING_SINCE_MS.putIfAbsent(idKey(item.dataItemId()), System.currentTimeMillis());
|
||||
IMMEDIATE_QUEUE.offer(item);
|
||||
scheduleImmediateDrain();
|
||||
}
|
||||
|
||||
private static boolean isOldEnoughForRetry(byte[] dataItemId, long nowMs) {
|
||||
Long pendingSince = PENDING_SINCE_MS.get(idKey(dataItemId));
|
||||
// Нет записи в памяти => pending пережил старт процесса и уже считается зависшим.
|
||||
return pendingSince == null || nowMs - pendingSince >= RETRY_MIN_PENDING_AGE_MS;
|
||||
}
|
||||
|
||||
private static String idKey(byte[] dataItemId) {
|
||||
return Base64.getUrlEncoder().withoutPadding().encodeToString(dataItemId);
|
||||
}
|
||||
|
||||
private static void scheduleImmediateDrain() {
|
||||
ScheduledExecutorService currentExecutor = executor;
|
||||
if (currentExecutor == null || currentExecutor.isShutdown()) return;
|
||||
if (!IMMEDIATE_DRAIN_SCHEDULED.compareAndSet(false, true)) return;
|
||||
try {
|
||||
currentExecutor.execute(ArweaveBlockPublisherScheduler::drainImmediateQueue);
|
||||
} catch (RuntimeException e) {
|
||||
IMMEDIATE_DRAIN_SCHEDULED.set(false);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private static void drainImmediateQueue() {
|
||||
try {
|
||||
ArweavePublishSignal.PendingRef item;
|
||||
while ((item = IMMEDIATE_QUEUE.poll()) != null) {
|
||||
try {
|
||||
boolean resolved = service.publishAcceptedNow(item);
|
||||
if (resolved) PENDING_SINCE_MS.remove(idKey(item.dataItemId()));
|
||||
} catch (Exception e) {
|
||||
// Turbo недоступен: не держим поток клиента и не молотим всю очередь одной и той же ошибкой.
|
||||
// Все элементы уже надёжно лежат в БД с pending=TRUE; плановый цикл подберёт их после min-age.
|
||||
log.warn("Immediate Turbo publish failed; pending blocks will be retried by the periodic cycle: {}", e.getMessage());
|
||||
IMMEDIATE_QUEUE.clear();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
IMMEDIATE_DRAIN_SCHEDULED.set(false);
|
||||
if (!IMMEDIATE_QUEUE.isEmpty()) scheduleImmediateDrain();
|
||||
}
|
||||
}
|
||||
|
||||
public static void close() {
|
||||
ArweavePublishSignal.clearListener();
|
||||
IMMEDIATE_QUEUE.clear();
|
||||
PENDING_SINCE_MS.clear();
|
||||
IMMEDIATE_DRAIN_SCHEDULED.set(false);
|
||||
ScheduledExecutorService e = executor;
|
||||
executor = null;
|
||||
service = null;
|
||||
config = null;
|
||||
STARTED.set(false);
|
||||
if (e != null) e.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
+51
-4
@@ -6,9 +6,12 @@ import shine.db.dao.BlocksDAO;
|
||||
import shine.db.dao.KeyRotationCandidateBlocksDAO;
|
||||
import shine.db.entities.BlockEntry;
|
||||
import shine.db.entities.KeyRotationCandidateBlockEntry;
|
||||
import server.sync.ArweavePublishSignal;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
/** Publishes locally-created signed user DataItems through the configured transport. */
|
||||
public final class ArweaveBlockPublisherService {
|
||||
@@ -26,7 +29,47 @@ public final class ArweaveBlockPublisherService {
|
||||
this.turboUploader = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO ? new TurboDataItemUploader(cfg) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Мгновенная попытка для только что принятого DataItem. Вызывается только из фонового executor-а:
|
||||
* сетевой запрос AddBlock не ждёт результат Turbo. Если запись уже успел обработать другой цикл, это no-op.
|
||||
* При ошибке pending-флаг в БД не меняется, поэтому обычный runCycle() повторит публикацию позже.
|
||||
*/
|
||||
public boolean publishAcceptedNow(ArweavePublishSignal.PendingRef pending) throws Exception {
|
||||
if (cfg.publishMode() != ArweaveBlocksConfig.PublishMode.TURBO) return false;
|
||||
byte[] id = pending.dataItemId();
|
||||
if (pending.kind() == ArweavePublishSignal.Kind.KEY_ROTATION_CANDIDATE) {
|
||||
KeyRotationCandidateBlockEntry entry = candidateBlocksDAO.getPendingArweaveByDataItemId(id);
|
||||
if (entry == null) return true;
|
||||
byte[] raw = entry.getBlockBytes();
|
||||
if (raw == null || raw.length == 0) return false;
|
||||
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
|
||||
candidateBlocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
|
||||
log.debug("Turbo immediately published key-rotation DataItem {} chain={} block={}",
|
||||
result.dataItemId(), entry.getCandidateBlockchainName(), entry.getBlockNumber());
|
||||
return true;
|
||||
}
|
||||
|
||||
BlockEntry entry = blocksDAO.getPendingArweaveByDataItemId(id);
|
||||
if (entry == null) return true;
|
||||
byte[] raw = entry.getBlockBytes();
|
||||
if (raw == null || raw.length == 0) return false;
|
||||
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
|
||||
blocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
|
||||
log.debug("Turbo immediately published SHiNE DataItem {} chain={} block={}",
|
||||
result.dataItemId(), entry.getBchName(), entry.getBlockNumber());
|
||||
return true;
|
||||
}
|
||||
|
||||
public int runCycle() throws Exception {
|
||||
return runCycle(id -> true, id -> {});
|
||||
}
|
||||
|
||||
/** Плановый retry может отфильтровать слишком свежие pending-записи и получить callback успешных id. */
|
||||
public int runRetryCycle(Predicate<byte[]> eligible, Consumer<byte[]> onPublished) throws Exception {
|
||||
return runCycle(eligible == null ? id -> true : eligible, onPublished == null ? id -> {} : onPublished);
|
||||
}
|
||||
|
||||
private int runCycle(Predicate<byte[]> eligible, Consumer<byte[]> onPublished) throws Exception {
|
||||
if (cfg.publishMode() == ArweaveBlocksConfig.PublishMode.NONE) return 0;
|
||||
|
||||
int maxItems = cfg.publishMaxItems();
|
||||
@@ -35,10 +78,11 @@ public final class ArweaveBlockPublisherService {
|
||||
|
||||
// Ротация приоритетна: пользователь уже заблокирован для обычной записи и ждёт завершения копирования.
|
||||
List<KeyRotationCandidateBlockEntry> rotationCandidates = candidateBlocksDAO.listPendingArweave(maxItems);
|
||||
rotationCandidates = rotationCandidates.stream().filter(e -> eligible.test(e.getDataItemId())).toList();
|
||||
if (!rotationCandidates.isEmpty()) {
|
||||
attempted += rotationCandidates.size();
|
||||
published += switch (cfg.publishMode()) {
|
||||
case TURBO -> publishTurboRotation(rotationCandidates);
|
||||
case TURBO -> publishTurboRotation(rotationCandidates, onPublished);
|
||||
case ARWEAVE -> publishDirectArweaveRotation(rotationCandidates);
|
||||
case NONE -> 0;
|
||||
};
|
||||
@@ -48,16 +92,17 @@ public final class ArweaveBlockPublisherService {
|
||||
if (remaining == 0) return published;
|
||||
|
||||
List<BlockEntry> candidates = blocksDAO.listPendingArweave(remaining);
|
||||
candidates = candidates.stream().filter(e -> eligible.test(e.getDataItemId())).toList();
|
||||
if (candidates.isEmpty()) return published;
|
||||
published += switch (cfg.publishMode()) {
|
||||
case TURBO -> publishTurbo(candidates);
|
||||
case TURBO -> publishTurbo(candidates, onPublished);
|
||||
case ARWEAVE -> publishDirectArweave(candidates);
|
||||
case NONE -> 0;
|
||||
};
|
||||
return published;
|
||||
}
|
||||
|
||||
private int publishTurboRotation(List<KeyRotationCandidateBlockEntry> candidates) throws Exception {
|
||||
private int publishTurboRotation(List<KeyRotationCandidateBlockEntry> candidates, Consumer<byte[]> onPublished) throws Exception {
|
||||
int published = 0;
|
||||
Exception firstFailure = null;
|
||||
for (KeyRotationCandidateBlockEntry e : candidates) {
|
||||
@@ -67,6 +112,7 @@ public final class ArweaveBlockPublisherService {
|
||||
try {
|
||||
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
|
||||
candidateBlocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
|
||||
onPublished.accept(id);
|
||||
published++;
|
||||
log.debug("Turbo published key-rotation DataItem {} chain={} block={}",
|
||||
result.dataItemId(), e.getCandidateBlockchainName(), e.getBlockNumber());
|
||||
@@ -81,7 +127,7 @@ public final class ArweaveBlockPublisherService {
|
||||
return published;
|
||||
}
|
||||
|
||||
private int publishTurbo(List<BlockEntry> candidates) throws Exception {
|
||||
private int publishTurbo(List<BlockEntry> candidates, Consumer<byte[]> onPublished) throws Exception {
|
||||
int published = 0;
|
||||
Exception firstFailure = null;
|
||||
for (BlockEntry e : candidates) {
|
||||
@@ -91,6 +137,7 @@ public final class ArweaveBlockPublisherService {
|
||||
try {
|
||||
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
|
||||
blocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
|
||||
onPublished.accept(id);
|
||||
published++;
|
||||
log.debug("Turbo published SHiNE DataItem {} chain={} block={}", result.dataItemId(), e.getBchName(), e.getBlockNumber());
|
||||
} catch (Exception ex) {
|
||||
|
||||
@@ -68,6 +68,16 @@ public final class BlocksDAO {
|
||||
}
|
||||
}
|
||||
|
||||
/** Возвращает ещё не опубликованный пользовательский DataItem для быстрой Turbo-попытки. */
|
||||
public BlockEntry getPendingArweaveByDataItemId(byte[] dataItemId) throws SQLException {
|
||||
if (dataItemId == null || dataItemId.length != 32) return null;
|
||||
String sql = baseSelect() + " WHERE data_item_id=? AND arweave_publish_pending=TRUE LIMIT 1";
|
||||
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
|
||||
ps.setBytes(1, dataItemId);
|
||||
try (ResultSet rs = ps.executeQuery()) { return rs.next() ? mapRow(rs) : null; }
|
||||
}
|
||||
}
|
||||
|
||||
public List<BlockEntry> listPendingArweave(int limit) throws SQLException {
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100_000));
|
||||
String sql = """
|
||||
|
||||
+14
@@ -132,6 +132,20 @@ public final class KeyRotationCandidateBlocksDAO {
|
||||
}
|
||||
}
|
||||
|
||||
/** Возвращает ещё не опубликованный candidate DataItem для быстрой Turbo-попытки. */
|
||||
public KeyRotationCandidateBlockEntry getPendingArweaveByDataItemId(byte[] dataItemId) throws SQLException {
|
||||
if (dataItemId == null || dataItemId.length != 32) return null;
|
||||
try (Connection c = db.getConnection();
|
||||
PreparedStatement ps = c.prepareStatement("""
|
||||
SELECT * FROM key_rotation_candidate_blocks
|
||||
WHERE data_item_id = ? AND arweave_publish_pending = TRUE
|
||||
LIMIT 1
|
||||
""")) {
|
||||
ps.setBytes(1, dataItemId);
|
||||
try (ResultSet rs = ps.executeQuery()) { return rs.next() ? map(rs) : null; }
|
||||
}
|
||||
}
|
||||
|
||||
/** Candidate-блоки имеют приоритет перед обычной очередью publisher-а. */
|
||||
public List<KeyRotationCandidateBlockEntry> listPendingArweave(int limit) throws SQLException {
|
||||
if (limit <= 0) return List.of();
|
||||
|
||||
+6
@@ -32,6 +32,7 @@ import server.logic.ws_protocol.JSON.handlers.channels.ChannelNamesStateBootstra
|
||||
import server.logic.ws_protocol.JSON.handlers.channels.UserCountersSupport;
|
||||
import server.logic.ws_protocol.WireCodes;
|
||||
import server.sync.AddBlockSyncService;
|
||||
import server.sync.ArweavePublishSignal;
|
||||
import shine.db.channels.ChannelNameRules;
|
||||
import shine.db.dao.BlockchainStateDAO;
|
||||
import shine.db.dao.BlocksDAO;
|
||||
@@ -720,6 +721,11 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
|
||||
upsertChat200StateFromCreate(chat200CreateSeed);
|
||||
}
|
||||
maybeApplyChat200Command(blockchainName, block);
|
||||
|
||||
if (publishToArweave) {
|
||||
// Только ставим быструю публикацию в очередь: ответ клиенту не ждёт сеть Turbo/Arweave.
|
||||
ArweavePublishSignal.userBlockAccepted(be.getDataItemId());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (isChannelSlugConflict(e)) {
|
||||
return new AddBlockResult(409, "channel_name_already_exists", serverLastNum, serverLastHashHex);
|
||||
|
||||
+5
@@ -13,6 +13,7 @@ import server.logic.ws_protocol.JSON.handlers.JsonMessageHandler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler_utils.BlockchainLocks;
|
||||
import server.logic.ws_protocol.JSON.handlers.keyRotation.entyties.Net_KeyRotationAddBlock_Request;
|
||||
import server.logic.ws_protocol.JSON.utils.NetExceptionResponseFactory;
|
||||
import server.sync.ArweavePublishSignal;
|
||||
import shine.db.DbController;
|
||||
import shine.db.dao.BlocksDAO;
|
||||
import shine.db.dao.KeyRotationCandidateBlocksDAO;
|
||||
@@ -164,6 +165,10 @@ public final class Net_KeyRotationAddBlock_Handler implements JsonMessageHandler
|
||||
c.setAutoCommit(oldAutoCommit);
|
||||
}
|
||||
|
||||
// Как и обычный пользовательский блок, candidate сразу ставится в неблокирующую Turbo-очередь.
|
||||
// При сетевой ошибке pending-флаг остаётся TRUE и его подберёт плановый retry.
|
||||
ArweavePublishSignal.keyRotationCandidateAccepted(entry.getDataItemId());
|
||||
|
||||
// progressCurrent обновляет Arweave publisher только после реальной публикации.
|
||||
KeyRotationSessionEntry refreshed = rotations.getActiveByLogin(login);
|
||||
return KeyRotationApiSupport.response(req.getOp(), req.getRequestId(), refreshed == null ? session : refreshed);
|
||||
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package server.sync;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Неблокирующий мост между приёмом блока по протоколу и Arweave/Turbo publisher-ом.
|
||||
*
|
||||
* Net-protocol не зависит от archive-модуля, поэтому publisher регистрирует здесь лёгкий listener.
|
||||
* Listener обязан только поставить ссылку на уже сохранённый DataItem в свою очередь и не выполнять
|
||||
* сетевую публикацию в потоке клиента.
|
||||
*/
|
||||
public final class ArweavePublishSignal {
|
||||
private static final Logger log = LoggerFactory.getLogger(ArweavePublishSignal.class);
|
||||
|
||||
public enum Kind {
|
||||
USER_BLOCK,
|
||||
KEY_ROTATION_CANDIDATE
|
||||
}
|
||||
|
||||
public record PendingRef(Kind kind, byte[] dataItemId) {
|
||||
public PendingRef {
|
||||
if (kind == null) throw new IllegalArgumentException("kind is null");
|
||||
if (dataItemId == null || dataItemId.length != 32) {
|
||||
throw new IllegalArgumentException("dataItemId must be 32 bytes");
|
||||
}
|
||||
dataItemId = Arrays.copyOf(dataItemId, dataItemId.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] dataItemId() {
|
||||
return Arrays.copyOf(dataItemId, dataItemId.length);
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Listener {
|
||||
void onPending(PendingRef item);
|
||||
}
|
||||
|
||||
private static volatile Listener listener;
|
||||
|
||||
private ArweavePublishSignal() {}
|
||||
|
||||
public static void setListener(Listener value) {
|
||||
listener = value;
|
||||
}
|
||||
|
||||
public static void clearListener() {
|
||||
listener = null;
|
||||
}
|
||||
|
||||
public static void userBlockAccepted(byte[] dataItemId) {
|
||||
publish(new PendingRef(Kind.USER_BLOCK, dataItemId));
|
||||
}
|
||||
|
||||
public static void keyRotationCandidateAccepted(byte[] dataItemId) {
|
||||
publish(new PendingRef(Kind.KEY_ROTATION_CANDIDATE, dataItemId));
|
||||
}
|
||||
|
||||
private static void publish(PendingRef item) {
|
||||
Listener current = listener;
|
||||
if (current == null) return;
|
||||
try {
|
||||
current.onPending(item);
|
||||
} catch (RuntimeException e) {
|
||||
// Ошибка постановки в быструю очередь не должна превращать уже принятый блок в ошибку клиента:
|
||||
// arweave_publish_pending остаётся TRUE, и плановый retry подберёт его позже.
|
||||
log.warn("Cannot enqueue immediate Arweave/Turbo publish; periodic retry will keep the block pending", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user