Добавить тестовые каналы и Arweave-синхронизацию

This commit is contained in:
AidarKC
2026-09-23 11:14:02 +03:00
parent 8b12760dde
commit ef707ec217
80 changed files with 15452 additions and 26 deletions
@@ -33,7 +33,10 @@ public final class ArweaveBlockSyncService {
private static final Base64.Decoder B64URL = Base64.getUrlDecoder();
private final ArweaveBlocksConfig cfg;
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(20)).build();
private final HttpClient http = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(20))
.followRedirects(HttpClient.Redirect.NORMAL)
.build();
private final ArweaveBlockImportDAO importDAO = ArweaveBlockImportDAO.getInstance();
private final BlocksDAO blocksDAO = BlocksDAO.getInstance();
private final SolanaUserPdaCurrentDAO usersDAO = SolanaUserPdaCurrentDAO.getInstance();
@@ -52,11 +55,10 @@ public final class ArweaveBlockSyncService {
long minHeight = Math.max(cfg.syncStartBlockHeight(), stored);
String cursor = null;
long maxHeight = minHeight;
Map<String, byte[]> rootCache = new HashMap<>();
int discovered = 0;
do {
JsonNode response = graphQlPage(minHeight, cursor);
JsonNode response = graphQlPage(minHeight, cursor, ArweaveBlocksConfig.TEST_TAG_VALUE + "-batch");
JsonNode txs = response.path("data").path("transactions");
if (response.has("errors")) throw new IOException("Arweave GraphQL errors: " + response.path("errors"));
JsonNode edges = txs.path("edges");
@@ -66,27 +68,23 @@ public final class ArweaveBlockSyncService {
for (JsonNode edge : edges) {
nextCursor = edge.path("cursor").asText(null);
JsonNode node = edge.path("node");
String itemIdText = node.path("id").asText("").trim();
String rootTx = node.path("bundledIn").path("id").asText("").trim();
String rootTx = node.path("id").asText("").trim();
long height = node.path("block").path("height").asLong(-1L);
if (itemIdText.isBlank() || rootTx.isBlank() || height < 0) continue;
if (rootTx.isBlank() || height < 0) continue;
maxHeight = Math.max(maxHeight, height);
byte[] id32;
try { id32 = B64URL.decode(itemIdText); }
catch (IllegalArgumentException bad) { continue; }
if (id32.length != 32 || blocksDAO.existsByDataItemId(id32) || importDAO.exists(id32)) continue;
byte[] bundle = rootCache.computeIfAbsent(rootTx, key -> {
try { return downloadRootBundle(key); }
catch (Exception e) { throw new RootDownloadRuntimeException(e); }
});
byte[] raw = Ans104Bundle.find(bundle, id32, Math.max(cfg.publishMaxItems() * 4, 100_000));
if (raw == null) throw new IOException("DataItem " + itemIdText + " not found in root bundle " + rootTx);
Ans104DataItem parsed = new Ans104DataItem(raw);
if (!parsed.hasTag(ArweaveBlocksConfig.TEST_TAG_NAME, ArweaveBlocksConfig.TEST_TAG_VALUE)) continue;
if (!parsed.verifySignature()) throw new IOException("Bad ANS-104 signature for " + itemIdText);
importDAO.enqueueIfMissing(id32, rootTx, height, raw, System.currentTimeMillis());
discovered++;
byte[] bundle = downloadRootBundle(rootTx);
for (Ans104Bundle.Entry entry : Ans104Bundle.read(bundle, Math.max(cfg.publishMaxItems() * 4, 100_000))) {
byte[] id32 = entry.id32();
if (id32 == null || id32.length != 32 || blocksDAO.existsByDataItemId(id32) || importDAO.exists(id32)) {
continue;
}
Ans104DataItem parsed = new Ans104DataItem(entry.rawDataItem());
if (!parsed.hasTag(ArweaveBlocksConfig.TEST_TAG_NAME, ArweaveBlocksConfig.TEST_TAG_VALUE)) continue;
if (!parsed.verifySignature()) throw new IOException("Bad ANS-104 signature in root bundle " + rootTx);
importDAO.enqueueIfMissing(id32, rootTx, height, entry.rawDataItem(), System.currentTimeMillis());
discovered++;
}
}
boolean hasNext = txs.path("pageInfo").path("hasNextPage").asBoolean(false);
cursor = hasNext ? nextCursor : null;
@@ -156,9 +154,9 @@ public final class ArweaveBlockSyncService {
stateDAO.insertIfMissing(s);
}
private JsonNode graphQlPage(long minHeight, String cursor) throws Exception {
private JsonNode graphQlPage(long minHeight, String cursor, String appTagValue) throws Exception {
String after = cursor == null ? "null" : "\"" + escapeGraphQl(cursor) + "\"";
String query = "query { transactions(tags:[{name:\"App\",values:[\"test5590\"]}], block:{min:" + minHeight + "}, first:" + cfg.syncPageSize() + ", after:" + after + ", sort:HEIGHT_ASC) { pageInfo { hasNextPage } edges { cursor node { id bundledIn { id } block { height } } } } }";
String query = "query { transactions(tags:[{name:\"App\",values:[\"" + escapeGraphQl(appTagValue) + "\"]}], block:{min:" + minHeight + "}, first:" + cfg.syncPageSize() + ", after:" + after + ", sort:HEIGHT_ASC) { pageInfo { hasNextPage } edges { cursor node { id block { height } } } } }";
String body = MAPPER.writeValueAsString(Map.of("query", query));
HttpRequest req = HttpRequest.newBuilder(URI.create(trim(cfg.syncGateway()) + "/graphql"))
.timeout(Duration.ofSeconds(60)).header("Content-Type","application/json").header("Accept","application/json")
@@ -181,5 +179,4 @@ public final class ArweaveBlockSyncService {
private static String trim(String s){return String.valueOf(s==null?"":s).trim().replaceAll("/+$","");}
private static String escapeGraphQl(String s){return s.replace("\\","\\\\").replace("\"","\\\"");}
private static final class RootDownloadRuntimeException extends RuntimeException { RootDownloadRuntimeException(Throwable cause){super(cause);} }
}
@@ -10,6 +10,7 @@ import shine.db.dao.SyncServersDAO;
import shine.db.entities.BlockEntry;
import shine.db.entities.SyncServerEntry;
import utils.blockchain.BlockchainNameUtil;
import utils.config.AppConfig;
import java.util.List;
import java.util.Locale;
@@ -46,6 +47,11 @@ public final class AddBlockSyncService {
private final SyncServersDAO syncServersDAO = SyncServersDAO.getInstance();
public void replicateAsync(String blockchainName, int blockNumber) {
if (!isEnabled()) {
log.debug("AddBlock sync skipped: blockchain.sync.enabled=false blockchainName={} blockNumber={}",
blockchainName, blockNumber);
return;
}
EXECUTOR.execute(() -> {
try {
replicate(blockchainName, blockNumber);
@@ -56,6 +62,10 @@ public final class AddBlockSyncService {
});
}
private static boolean isEnabled() {
return AppConfig.getInstance().getBoolean("blockchain.sync.enabled", true);
}
private void replicate(String blockchainName, int blockNumber) throws Exception {
String ownerLogin = normalize(BlockchainNameUtil.loginFromBlockchainName(blockchainName));
if (ownerLogin == null) {
@@ -327,6 +327,25 @@ public final class SolanaUsersSyncService
state.lastSeenSignature()
);
if (state.lastSeenSignature() == null) {
log.warn(
"History checkpoint is empty. Running current-state full snapshot bootstrap instead of replaying all historical transactions."
);
runFullSnapshotFallback(
state,
fetchResult,
nowMs
);
markReadyAfterSync(
state,
nowMs
);
return;
}
if (state.lastSeenSignature() != null
&& !fetchResult.anchorFound()) {
@@ -16,6 +16,7 @@ import shine.db.entities.SolanaUserPdaCurrentEntry;
import shine.db.entities.SyncServerEntry;
import server.sync.BlockchainResyncGuard;
import utils.blockchain.BlockchainNameUtil;
import utils.config.AppConfig;
import java.util.concurrent.locks.ReentrantLock;
import java.util.List;
@@ -56,6 +57,10 @@ public final class PeriodicBlockchainSyncService {
private PeriodicBlockchainSyncService() {}
public static void startOrLog() {
if (!isEnabled()) {
log.info("Periodic blockchain sync disabled by blockchain.sync.enabled=false");
return;
}
if (!STARTED.compareAndSet(false, true)) {
return;
}
@@ -68,6 +73,10 @@ public final class PeriodicBlockchainSyncService {
log.info("Periodic blockchain sync scheduled: startup + every {} hours", PERIOD_HOURS);
}
private static boolean isEnabled() {
return AppConfig.getInstance().getBoolean("blockchain.sync.enabled", true);
}
private static void runCycleSafe() {
try {
runCycle();
@@ -27,6 +27,13 @@ solana.users.sync.pollIntervalSeconds=300
# ------------------------------------------------------------
sync.importUserProfileFromPartner.enabled=false
# ------------------------------------------------------------
# Прямая межсерверная синхронизация пользовательских блокчейнов
# через sync_servers: ListBlockchainHeads/GetBlockchainBlock -> локальный AddBlock.
# Для проверки режима Solana + Arweave без прямых связей серверов можно выключить.
# ------------------------------------------------------------
blockchain.sync.enabled=true
# ------------------------------------------------------------
# Server public info
# Эти поля используются JSON-операцией GetServerInfo.