SHA256
Добавить импорт архивов с доверенных серверов
This commit is contained in:
+37
@@ -0,0 +1,37 @@
|
||||
package server.archive;
|
||||
|
||||
import utils.config.AppConfig;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
|
||||
/** Настройки чтения архивов других доверенных SHiNE-серверов. Пустой whitelist = импорт выключен. */
|
||||
public record ArchiveImportConfig(
|
||||
Set<String> allowedPublishers,
|
||||
int intervalMinutes,
|
||||
String arweaveGateway,
|
||||
Path workDir
|
||||
) {
|
||||
public static ArchiveImportConfig load() {
|
||||
AppConfig c = AppConfig.getInstance();
|
||||
Set<String> approved = new LinkedHashSet<>();
|
||||
String raw = c.getParam("archive.import.allowedPublishers");
|
||||
if (raw != null) {
|
||||
for (String item : raw.split(",")) {
|
||||
String login = item.trim().toLowerCase(Locale.ROOT);
|
||||
if (!login.isEmpty()) approved.add(login);
|
||||
}
|
||||
}
|
||||
String gateway = c.getParam("archive.arweave.gateway");
|
||||
if (gateway == null || gateway.isBlank()) gateway = "https://arweave.net";
|
||||
String dir = c.getParam("archive.import.workDir");
|
||||
if (dir == null || dir.isBlank()) dir = "data/archive-import";
|
||||
return new ArchiveImportConfig(
|
||||
Set.copyOf(approved),
|
||||
Math.max(1, c.getInt("archive.import.intervalMinutes", 60)),
|
||||
gateway.trim(),
|
||||
Path.of(dir));
|
||||
}
|
||||
|
||||
public boolean enabled() { return !allowedPublishers.isEmpty(); }
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package server.archive;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.concurrent.*;
|
||||
|
||||
/**
|
||||
* Фоновая проверка уже синхронизированной локальной таблицы User PDA.
|
||||
* Solana отдельно здесь не опрашивается: первый запуск через 10 секунд после старта,
|
||||
* затем по умолчанию раз в час.
|
||||
*/
|
||||
public final class ArchiveImportScheduler {
|
||||
private static final Logger log = LoggerFactory.getLogger(ArchiveImportScheduler.class);
|
||||
private static ScheduledExecutorService executor;
|
||||
|
||||
private ArchiveImportScheduler() {}
|
||||
|
||||
public static synchronized void startOrLog() {
|
||||
if (executor != null) return;
|
||||
ArchiveImportConfig cfg = ArchiveImportConfig.load();
|
||||
if (!cfg.enabled()) {
|
||||
log.info("Archive importer выключен: archive.import.allowedPublishers пуст");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ArchiveImportService service = new ArchiveImportService(cfg);
|
||||
executor = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "archive-importer");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
executor.scheduleWithFixedDelay(
|
||||
() -> {
|
||||
try { service.pollOnce(); }
|
||||
catch (Exception e) { log.error("Archive importer cycle failed", e); }
|
||||
},
|
||||
10,
|
||||
Math.multiplyExact((long) cfg.intervalMinutes(), 60L),
|
||||
TimeUnit.SECONDS);
|
||||
log.info("Archive importer включён. approvedPublishers={} interval={}min",
|
||||
cfg.allowedPublishers(), cfg.intervalMinutes());
|
||||
} catch (Exception e) {
|
||||
log.error("Archive importer не запущен", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static synchronized void close() {
|
||||
if (executor != null) {
|
||||
executor.shutdownNow();
|
||||
executor = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
package server.archive;
|
||||
|
||||
import blockchain.BchBlockEntry;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler_utils.BlockchainLocks;
|
||||
import shine.db.archive.*;
|
||||
import shine.db.dao.*;
|
||||
import shine.db.entities.BlockchainStateEntry;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.*;
|
||||
import java.nio.file.*;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/** Импорт archive head только от серверов из server-side whitelist. */
|
||||
public final class ArchiveImportService {
|
||||
private static final Logger log = LoggerFactory.getLogger(ArchiveImportService.class);
|
||||
private final ArchiveImportConfig cfg;
|
||||
private final ArchiveImportDAO dao = ArchiveImportDAO.getInstance();
|
||||
private final BlockchainStateDAO stateDao = BlockchainStateDAO.getInstance();
|
||||
private final BlocksDAO blocksDao = BlocksDAO.getInstance();
|
||||
private final ShineArchiveReader reader = new ShineArchiveReader();
|
||||
private final Net_AddBlock_Handler addBlockHandler = new Net_AddBlock_Handler();
|
||||
private final HttpClient http = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(20))
|
||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||
.build();
|
||||
|
||||
public ArchiveImportService(ArchiveImportConfig cfg) throws IOException {
|
||||
this.cfg = Objects.requireNonNull(cfg);
|
||||
Files.createDirectories(cfg.workDir());
|
||||
}
|
||||
|
||||
/**
|
||||
* Никаких отдельных Solana RPC запросов здесь нет.
|
||||
* Читаем только solana_user_pda_current, которую уже обновляет обычный Solana users sync.
|
||||
*/
|
||||
public void pollOnce() throws Exception {
|
||||
if (!cfg.enabled()) return;
|
||||
for (ArchivePublisherHead head : dao.listPendingPublisherHeads(cfg.allowedPublishers())) {
|
||||
try {
|
||||
importHead(head);
|
||||
} catch (Exception e) {
|
||||
log.error("Archive import publisher={} tx={} failed", head.login(), head.archiveHeadTxId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void importHead(ArchivePublisherHead publisher) throws Exception {
|
||||
String headTxId = publisher.archiveHeadTxId();
|
||||
byte[] headHash = hex(publisher.archiveHeadHash());
|
||||
|
||||
// Если процесс успел импортировать head, но упал до установки archive_imported=true,
|
||||
// следующий цикл просто завершает отметку без повторного скачивания всей истории.
|
||||
if (headTxId.equals(publisher.lastImportedArchiveTxId())) {
|
||||
if (!dao.markHeadImported(publisher.login(), headTxId)) {
|
||||
log.info("Archive head changed while finalizing import marker publisher={} oldHead={}",
|
||||
publisher.login(), headTxId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
Path headFile = download(headTxId);
|
||||
ShineArchiveReader.Document head;
|
||||
try {
|
||||
head = reader.readAndVerify(headFile, publisher.login(), headHash, publisher.rootKey());
|
||||
} finally {
|
||||
Files.deleteIfExists(headFile);
|
||||
}
|
||||
|
||||
// FULL reference table позволяет догнать все archive blocks, пропущенные пока сервер был выключен.
|
||||
List<ShineArchiveReader.Reference> previous = new ArrayList<>(head.references());
|
||||
previous.sort(Comparator.comparingLong(ShineArchiveReader.Reference::bigBlockNumber));
|
||||
|
||||
int startIndex = findResumeIndex(previous, publisher.lastImportedArchiveTxId(), headTxId);
|
||||
for (int i = startIndex; i < previous.size(); i++) {
|
||||
ShineArchiveReader.Reference ref = previous.get(i);
|
||||
importOne(publisher, ref.arweaveTxId(), ref.archiveHash());
|
||||
if (!dao.advanceImportCursor(publisher.login(), headTxId, ref.arweaveTxId())) {
|
||||
throw new IllegalStateException("Archive head changed during import for publisher " + publisher.login());
|
||||
}
|
||||
}
|
||||
|
||||
// Head уже скачан и проверен выше — второй раз его из Arweave не качаем.
|
||||
importDocument(publisher, headTxId, head);
|
||||
if (!dao.advanceImportCursor(publisher.login(), headTxId, headTxId)) {
|
||||
throw new IllegalStateException("Archive head changed during final import for publisher " + publisher.login());
|
||||
}
|
||||
if (!dao.markHeadImported(publisher.login(), headTxId)) {
|
||||
throw new IllegalStateException("Archive head changed before imported marker for publisher " + publisher.login());
|
||||
}
|
||||
|
||||
log.info("Archive head fully imported publisher={} bigBlock={} tx={}",
|
||||
publisher.login(), head.bigBlockNumber(), headTxId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Возвращает индекс первого ещё не импортированного previous big block.
|
||||
* Пустой cursor = новый сервер, импортируем всю FULL историю.
|
||||
*/
|
||||
private static int findResumeIndex(List<ShineArchiveReader.Reference> previous,
|
||||
String lastImportedTxId,
|
||||
String currentHeadTxId) {
|
||||
String last = lastImportedTxId == null ? "" : lastImportedTxId.trim();
|
||||
if (last.isEmpty()) return 0;
|
||||
if (last.equals(currentHeadTxId)) return previous.size();
|
||||
for (int i = 0; i < previous.size(); i++) {
|
||||
if (last.equals(previous.get(i).arweaveTxId())) return i + 1;
|
||||
}
|
||||
throw new IllegalStateException(
|
||||
"Последний импортированный archive TX не найден в FULL history текущего head: " + last);
|
||||
}
|
||||
|
||||
private void importOne(ArchivePublisherHead publisher, String txId, byte[] expectedHash) throws Exception {
|
||||
Path file = download(txId);
|
||||
try {
|
||||
ShineArchiveReader.Document doc = reader.readAndVerify(
|
||||
file, publisher.login(), expectedHash, publisher.rootKey());
|
||||
importDocument(publisher, txId, doc);
|
||||
} finally {
|
||||
Files.deleteIfExists(file);
|
||||
}
|
||||
}
|
||||
|
||||
private void importDocument(ArchivePublisherHead publisher, String txId,
|
||||
ShineArchiveReader.Document doc) throws Exception {
|
||||
for (ShineArchiveReader.Chunk chunk : doc.chunks()) {
|
||||
processChunk(publisher.login(), txId, doc, chunk);
|
||||
}
|
||||
log.info("Archive imported publisher={} bigBlock={} tx={} chunks={}",
|
||||
publisher.login(), doc.bigBlockNumber(), txId, doc.chunks().size());
|
||||
}
|
||||
|
||||
private void processChunk(String publisherLogin, String txId,
|
||||
ShineArchiveReader.Document doc,
|
||||
ShineArchiveReader.Chunk chunk) throws Exception {
|
||||
ReentrantLock lock = BlockchainLocks.lockFor(chunk.blockchainName());
|
||||
lock.lock();
|
||||
try {
|
||||
ensureState(chunk.blockchainName());
|
||||
BlockchainStateEntry state = stateDao.getByBlockchainName(chunk.blockchainName());
|
||||
int localLast = state == null ? -1 : state.getLastBlockNumber();
|
||||
long archiveLast = -1;
|
||||
|
||||
for (byte[] raw : chunk.rawRecords()) {
|
||||
BchBlockEntry block = new BchBlockEntry(raw);
|
||||
archiveLast = Integer.toUnsignedLong(block.blockNumber);
|
||||
|
||||
if (block.blockNumber <= localLast) {
|
||||
byte[] localHash = blocksDao.getHashByNumber(chunk.blockchainName(), block.blockNumber);
|
||||
if (!Arrays.equals(localHash, block.getHash32())) {
|
||||
throw new IllegalStateException(
|
||||
"Archive conflict " + chunk.blockchainName() + "#" + block.blockNumber);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (block.blockNumber != localLast + 1) {
|
||||
throw new IllegalStateException(
|
||||
"Archive gap " + chunk.blockchainName() + ": local=" + localLast +
|
||||
" next=" + block.blockNumber);
|
||||
}
|
||||
|
||||
var result = addBlockHandler.addBlockFromArchive(chunk.blockchainName(), raw);
|
||||
if (!result.ok()) {
|
||||
throw new IllegalStateException(
|
||||
"Archive AddBlock rejected " + chunk.blockchainName() + "#" +
|
||||
block.blockNumber + ": " + result.reasonCode());
|
||||
}
|
||||
localLast = block.blockNumber;
|
||||
}
|
||||
|
||||
if (archiveLast >= 0) {
|
||||
dao.upsertLocation(new ArchiveBlockchainLocation(
|
||||
chunk.blockchainName(), publisherLogin, txId, doc.archiveHash(), doc.bigBlockNumber(),
|
||||
chunk.offset(), chunk.size(), archiveLast, System.currentTimeMillis()));
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureState(String blockchainName) throws Exception {
|
||||
if (stateDao.getByBlockchainName(blockchainName) != null) return;
|
||||
ArchiveImportDAO.BlockchainIdentity id = dao.getBlockchainIdentity(blockchainName);
|
||||
if (id == null) throw new IllegalStateException("Нет User PDA для blockchain " + blockchainName);
|
||||
|
||||
BlockchainStateEntry s = new BlockchainStateEntry();
|
||||
s.setBlockchainName(id.blockchainName());
|
||||
s.setLogin(id.login());
|
||||
s.setBlockchainKey(id.blockchainKey());
|
||||
s.setSizeLimit(id.sizeLimit());
|
||||
s.setFileSizeBytes(0);
|
||||
s.setLastBlockNumber(-1);
|
||||
s.setLastBlockHash(null);
|
||||
s.setUpdatedAtMs(System.currentTimeMillis());
|
||||
stateDao.insertIfMissing(s);
|
||||
}
|
||||
|
||||
private Path download(String txId) throws Exception {
|
||||
String gateway = cfg.arweaveGateway().replaceAll("/+$", "");
|
||||
Path target = Files.createTempFile(cfg.workDir(), "archive-", ".download");
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create(gateway + "/" + txId))
|
||||
.timeout(Duration.ofMinutes(10))
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<Path> resp = http.send(req, HttpResponse.BodyHandlers.ofFile(target));
|
||||
if (resp.statusCode() < 200 || resp.statusCode() >= 300) {
|
||||
Files.deleteIfExists(target);
|
||||
throw new IOException("Arweave HTTP " + resp.statusCode() + " tx=" + txId);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
private static byte[] hex(String value) {
|
||||
String s = String.valueOf(value == null ? "" : value).trim();
|
||||
if (s.length() != 64) throw new IllegalArgumentException("archive_head_hash должен быть HEX64");
|
||||
byte[] out = new byte[32];
|
||||
for (int i = 0; i < 32; i++) {
|
||||
out[i] = (byte) Integer.parseInt(s.substring(i * 2, i * 2 + 2), 16);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
+39
@@ -8,6 +8,7 @@ import shine.db.archive.ArchiveChainCursor;
|
||||
import shine.db.archive.ArchivePublishJob;
|
||||
import shine.db.archive.ArchivePublishJobChain;
|
||||
import shine.db.dao.ArchivePublicationDAO;
|
||||
import shine.db.dao.ArchiveImportDAO;
|
||||
import shine.db.dao.BlockchainStateDAO;
|
||||
import shine.db.dao.BlocksDAO;
|
||||
import shine.db.entities.BlockEntry;
|
||||
@@ -24,6 +25,7 @@ public final class ArchivePublisherService {
|
||||
|
||||
private final ArchivePublisherConfig cfg;
|
||||
private final ArchivePublicationDAO archiveDao = ArchivePublicationDAO.getInstance();
|
||||
private final ArchiveImportDAO archiveImportDao = ArchiveImportDAO.getInstance();
|
||||
private final BlockchainStateDAO stateDao = BlockchainStateDAO.getInstance();
|
||||
private final BlocksDAO blocksDao = BlocksDAO.getInstance();
|
||||
private final ShineArchiveWriter writer = new ShineArchiveWriter();
|
||||
@@ -40,6 +42,32 @@ public final class ArchivePublisherService {
|
||||
this.arweave = new ArweaveArchiveService(cfg);
|
||||
this.solana = new SolanaArchiveHeadWriter(cfg);
|
||||
this.solana.validateKeysAgainstPda();
|
||||
backfillOwnLocationIndex();
|
||||
}
|
||||
|
||||
/**
|
||||
* После обновления старого publisher-сервера заполняет универсальный location index
|
||||
* из уже существующих finalized cursors, не дожидаясь следующего суточного archive block.
|
||||
*/
|
||||
private void backfillOwnLocationIndex() throws Exception {
|
||||
Map<Long, ArchiveBigBlockRef> finalized = new HashMap<>();
|
||||
for (ArchiveBigBlockRef ref : archiveDao.listFinalizedBigBlocks()) finalized.put(ref.bigBlockNumber(), ref);
|
||||
if (finalized.isEmpty()) return;
|
||||
long now = System.currentTimeMillis();
|
||||
int updated = 0;
|
||||
for (BlockchainStateEntry state : stateDao.listAll()) {
|
||||
if (state == null || state.getBlockchainName() == null || state.getBlockchainName().isBlank()) continue;
|
||||
ArchiveChainCursor cursor = archiveDao.getCursor(state.getBlockchainName());
|
||||
if (cursor == null) continue;
|
||||
ArchiveBigBlockRef ref = finalized.get(cursor.lastArchiveBigBlockNumber());
|
||||
if (ref == null || ref.arweaveTxId() == null || ref.arweaveTxId().length != 32) continue;
|
||||
archiveImportDao.upsertLocation(new shine.db.archive.ArchiveBlockchainLocation(
|
||||
cursor.blockchainName(), cfg.serverLogin(), b64url(ref.arweaveTxId()),
|
||||
cursor.lastArchiveBigBlockHash(), cursor.lastArchiveBigBlockNumber(),
|
||||
cursor.lastChunkOffset(), cursor.lastChunkSize(), cursor.lastArchivedSourceBlockNumber(), now));
|
||||
updated++;
|
||||
}
|
||||
if (updated > 0) log.info("Archive publisher: восстановлен location index для {} blockchain", updated);
|
||||
}
|
||||
|
||||
/** После рестарта продолжает только уже существующий незавершённый job; новый snapshot не создаёт. */
|
||||
@@ -285,6 +313,17 @@ public final class ArchivePublisherService {
|
||||
|
||||
private void commitCursors(ArchivePublishJob job) throws Exception {
|
||||
archiveDao.commitCursors(job.id(), job.archiveHash());
|
||||
|
||||
// Тот же универсальный индекс используется и для собственных опубликованных архивов.
|
||||
// Поэтому UI publisher-сервера получает ссылку даже если он не импортирует самого себя через whitelist.
|
||||
String txId = b64url(job.arweaveTxId());
|
||||
long now = System.currentTimeMillis();
|
||||
for (ArchivePublishJobChain ch : archiveDao.listJobChains(job.id())) {
|
||||
if (ch.newChunkOffset() == null || ch.newChunkSize() == null) continue;
|
||||
archiveImportDao.upsertLocation(new shine.db.archive.ArchiveBlockchainLocation(
|
||||
ch.blockchainName(), cfg.serverLogin(), txId, job.archiveHash(), job.bigBlockNumber(),
|
||||
ch.newChunkOffset(), ch.newChunkSize(), ch.toSourceBlockNumber(), now));
|
||||
}
|
||||
}
|
||||
|
||||
private static Path requireLocalFile(ArchivePublishJob job) {
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package server.archive;
|
||||
|
||||
import sync.util.Base58Util;
|
||||
import utils.crypto.Ed25519Util;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.*;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.*;
|
||||
|
||||
/** Строгий reader/verifier формата SHINE-ARCHIVE v1.0. */
|
||||
public final class ShineArchiveReader {
|
||||
public record Reference(long bigBlockNumber, byte[] archiveHash, String arweaveTxId) {}
|
||||
public record Chunk(long offset, long size, String blockchainName, List<byte[]> rawRecords,
|
||||
long previousBigBlockRef, long previousChunkOffset, long previousChunkSize) {}
|
||||
public record Document(long bigBlockNumber, long createdAtMs, String creatorLogin,
|
||||
List<Reference> references, List<Chunk> chunks, byte[] archiveHash) {}
|
||||
|
||||
public Document readAndVerify(Path file, String expectedPublisherLogin, byte[] expectedHash, String rootPublicKeyBase58) throws Exception {
|
||||
long fileSize = Files.size(file);
|
||||
if (fileSize < 13 + 38 + 1 + 32 + 64) throw new IOException("SHINE-ARCHIVE слишком короткий");
|
||||
if (fileSize >= 0x1_0000_0000L) throw new IOException("SHINE-ARCHIVE v1 >= 4 GiB не поддерживается");
|
||||
|
||||
byte[] computedHash = digestPrefix(file, fileSize - 96);
|
||||
try (RandomAccessFile in = new RandomAccessFile(file.toFile(), "r")) {
|
||||
byte[] magic = new byte[ShineArchiveWriter.MAGIC.length];
|
||||
in.readFully(magic);
|
||||
if (!Arrays.equals(magic, ShineArchiveWriter.MAGIC)) throw new IOException("Bad SHINE-ARCHIVE magic");
|
||||
int major=in.readUnsignedByte(), minor=in.readUnsignedByte();
|
||||
if (major != 1 || minor != 0) throw new IOException("Unsupported SHINE-ARCHIVE version " + major + "." + minor);
|
||||
long headerSize=u32(in); long bigBlock=u32(in); long createdAt=in.readLong();
|
||||
int creatorLen=in.readUnsignedByte(); int referencesMode=in.readUnsignedByte();
|
||||
long refsCount=u32(in); int refSize=in.readUnsignedShort(); long parentRef=u32(in);
|
||||
long chunksCount=u32(in); long recordsCount=u32(in);
|
||||
if (referencesMode != ShineArchiveWriter.REFERENCES_FULL || refSize != ShineArchiveWriter.REFERENCE_ENTRY_SIZE) {
|
||||
throw new IOException("Unsupported references table");
|
||||
}
|
||||
String creator=readUtf8(in,creatorLen);
|
||||
if (!creator.equalsIgnoreCase(String.valueOf(expectedPublisherLogin))) {
|
||||
throw new SecurityException("Archive creator " + creator + " != trusted publisher " + expectedPublisherLogin);
|
||||
}
|
||||
if (headerSize != in.getFilePointer()) throw new IOException("Некорректный header_size");
|
||||
if (refsCount > 10_000_000L || chunksCount > 10_000_000L) throw new IOException("Слишком много refs/chunks");
|
||||
|
||||
List<Reference> refs=new ArrayList<>((int)Math.min(refsCount, Integer.MAX_VALUE));
|
||||
for(long i=0;i<refsCount;i++) {
|
||||
long number=u32(in); byte[] hash=readFixed(in,32); byte[] tx=readFixed(in,32);
|
||||
refs.add(new Reference(number,hash,Base64.getUrlEncoder().withoutPadding().encodeToString(tx)));
|
||||
}
|
||||
if (refs.isEmpty() && parentRef != ShineArchiveWriter.NO_REFERENCE_U32) throw new IOException("Некорректный parent reference");
|
||||
if (!refs.isEmpty() && parentRef >= refs.size()) throw new IOException("parent_reference_index вне таблицы");
|
||||
|
||||
List<Chunk> chunks=new ArrayList<>((int)Math.min(chunksCount, Integer.MAX_VALUE));
|
||||
long parsedRecords=0;
|
||||
for(long ci=0;ci<chunksCount;ci++) {
|
||||
long chunkOffset=in.getFilePointer(); long chunkSize=u32(in); long chunkEnd=chunkOffset+chunkSize;
|
||||
if (chunkSize < 21 || chunkEnd > fileSize-96) throw new IOException("Некорректный chunk_size");
|
||||
int nameLen=in.readUnsignedByte(); String bch=readUtf8(in,nameLen); long count=u32(in);
|
||||
if (count > 10_000_000L) throw new IOException("Слишком много records в chunk " + bch);
|
||||
List<byte[]> records=new ArrayList<>((int)Math.min(count,Integer.MAX_VALUE));
|
||||
for(long ri=0;ri<count;ri++) {
|
||||
long n=u32(in); if(n<=0 || n>Integer.MAX_VALUE || in.getFilePointer()+n>chunkEnd-12) throw new IOException("Некорректный record_size");
|
||||
records.add(readFixed(in,(int)n)); parsedRecords++;
|
||||
}
|
||||
long prevRef=u32(in), prevOffset=u32(in), prevSize=u32(in);
|
||||
if (prevRef != ShineArchiveWriter.NO_REFERENCE_U32 && prevRef >= refs.size()) throw new IOException("previous_big_block_ref вне таблицы");
|
||||
if (in.getFilePointer()!=chunkEnd) throw new IOException("Chunk framing mismatch: " + bch);
|
||||
chunks.add(new Chunk(chunkOffset,chunkSize,bch,List.copyOf(records),prevRef,prevOffset,prevSize));
|
||||
}
|
||||
if (parsedRecords != recordsCount) throw new IOException("total_user_records_count mismatch");
|
||||
|
||||
int closerLen=in.readUnsignedByte(); String closer=readUtf8(in,closerLen);
|
||||
if(!closer.equals(creator)) throw new SecurityException("creator_login != closer_login");
|
||||
if(in.getFilePointer()!=fileSize-96) throw new IOException("Некорректная граница footer/hash");
|
||||
byte[] storedHash=readFixed(in,32); byte[] signature=readFixed(in,64);
|
||||
if(in.getFilePointer()!=fileSize) throw new IOException("Лишние данные после signature");
|
||||
if(!Arrays.equals(storedHash,computedHash)) throw new SecurityException("SHA-256 archive не совпадает с footer");
|
||||
if(expectedHash!=null && !Arrays.equals(expectedHash,computedHash)) throw new SecurityException("SHA-256 archive не совпадает с ожидаемым hash");
|
||||
byte[] root=Base58Util.decode(rootPublicKeyBase58);
|
||||
if(root.length!=32) throw new SecurityException("Некорректный root public key publisher-а");
|
||||
byte[] signed=new byte[ShineArchiveWriter.SIGNATURE_DOMAIN.length+32];
|
||||
System.arraycopy(ShineArchiveWriter.SIGNATURE_DOMAIN,0,signed,0,ShineArchiveWriter.SIGNATURE_DOMAIN.length);
|
||||
System.arraycopy(computedHash,0,signed,ShineArchiveWriter.SIGNATURE_DOMAIN.length,32);
|
||||
if(!Ed25519Util.verify(signed,signature,root)) throw new SecurityException("Не прошла подпись SHINE-ARCHIVE publisher-а");
|
||||
return new Document(bigBlock,createdAt,creator,List.copyOf(refs),List.copyOf(chunks),computedHash);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] digestPrefix(Path file,long bytes) throws Exception {
|
||||
MessageDigest md=MessageDigest.getInstance("SHA-256");
|
||||
try(InputStream in=new BufferedInputStream(Files.newInputStream(file))) {
|
||||
byte[] buf=new byte[1024*1024]; long left=bytes;
|
||||
while(left>0){int n=in.read(buf,0,(int)Math.min(buf.length,left)); if(n<0) throw new EOFException(); md.update(buf,0,n); left-=n;}
|
||||
}
|
||||
return md.digest();
|
||||
}
|
||||
private static long u32(RandomAccessFile in) throws IOException { return Integer.toUnsignedLong(in.readInt()); }
|
||||
private static byte[] readFixed(RandomAccessFile in,int n) throws IOException { byte[] b=new byte[n]; in.readFully(b); return b; }
|
||||
private static String readUtf8(RandomAccessFile in,int n) throws IOException { return new String(readFixed(in,n), StandardCharsets.UTF_8); }
|
||||
}
|
||||
@@ -39,6 +39,7 @@ public final class DatabaseInitializer {
|
||||
public static final int SCHEMA_VERSION_20 = 20;
|
||||
public static final int SCHEMA_VERSION_21 = 21;
|
||||
public static final int SCHEMA_VERSION_22 = 22;
|
||||
public static final int SCHEMA_VERSION_23 = 23;
|
||||
public static final String POSTGRES_SCHEMA_RESOURCE = "postgres/schema_v1.sql";
|
||||
public static final String POSTGRES_MIGRATION_V2_RESOURCE = "postgres/migration_v2.sql";
|
||||
public static final String POSTGRES_MIGRATION_V3_RESOURCE = "postgres/migration_v3.sql";
|
||||
@@ -61,6 +62,7 @@ public final class DatabaseInitializer {
|
||||
public static final String POSTGRES_MIGRATION_V20_RESOURCE = "postgres/migration_v20.sql";
|
||||
public static final String POSTGRES_MIGRATION_V21_RESOURCE = "postgres/migration_v21.sql";
|
||||
public static final String POSTGRES_MIGRATION_V22_RESOURCE = "postgres/migration_v22.sql";
|
||||
public static final String POSTGRES_MIGRATION_V23_RESOURCE = "postgres/migration_v23.sql";
|
||||
|
||||
private DatabaseInitializer() {}
|
||||
|
||||
@@ -218,6 +220,10 @@ public final class DatabaseInitializer {
|
||||
runSqlScript(conn, POSTGRES_MIGRATION_V22_RESOURCE);
|
||||
currentVersion = SCHEMA_VERSION_22;
|
||||
}
|
||||
if (currentVersion < SCHEMA_VERSION_23) {
|
||||
runSqlScript(conn, POSTGRES_MIGRATION_V23_RESOURCE);
|
||||
currentVersion = SCHEMA_VERSION_23;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package shine.db.archive;
|
||||
|
||||
/** Последний известный архивный chunk конкретной blockchain на этом сервере. */
|
||||
public record ArchiveBlockchainLocation(
|
||||
String blockchainName,
|
||||
String publisherLogin,
|
||||
String arweaveTxId,
|
||||
byte[] archiveHash,
|
||||
long bigBlockNumber,
|
||||
long chunkOffset,
|
||||
long chunkSize,
|
||||
long sourceLastBlockNumber,
|
||||
long updatedAtMs
|
||||
) {}
|
||||
@@ -0,0 +1,14 @@
|
||||
package shine.db.archive;
|
||||
|
||||
/**
|
||||
* Текущая archive head trusted publisher-а из локальной таблицы,
|
||||
* которую уже заполнил обычный Solana users sync.
|
||||
*/
|
||||
public record ArchivePublisherHead(
|
||||
String login,
|
||||
String rootKey,
|
||||
String archiveHeadTxId,
|
||||
String archiveHeadHash,
|
||||
boolean archiveImported,
|
||||
String lastImportedArchiveTxId
|
||||
) {}
|
||||
@@ -0,0 +1,150 @@
|
||||
package shine.db.dao;
|
||||
|
||||
import shine.db.DbController;
|
||||
import shine.db.archive.ArchiveBlockchainLocation;
|
||||
import shine.db.archive.ArchivePublisherHead;
|
||||
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
|
||||
/** БД индекс последних archive chunks и локальное состояние импорта archive head. */
|
||||
public final class ArchiveImportDAO {
|
||||
private static volatile ArchiveImportDAO instance;
|
||||
private final DbController db = DbController.getInstance();
|
||||
|
||||
private ArchiveImportDAO() {}
|
||||
|
||||
public static ArchiveImportDAO getInstance() {
|
||||
if (instance == null) synchronized (ArchiveImportDAO.class) {
|
||||
if (instance == null) instance = new ArchiveImportDAO();
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Возвращает только доверенных publisher-ов, для которых обычный Solana users sync
|
||||
* уже увидел archive head, но этот head ещё не был полностью импортирован локально.
|
||||
*/
|
||||
public List<ArchivePublisherHead> listPendingPublisherHeads(Set<String> approvedLogins) throws SQLException {
|
||||
if (approvedLogins == null || approvedLogins.isEmpty()) return List.of();
|
||||
List<ArchivePublisherHead> out = new ArrayList<>();
|
||||
String sql = """
|
||||
SELECT login, root_key, archive_head_tx_id, archive_head_hash,
|
||||
archive_imported, archive_last_imported_tx_id
|
||||
FROM solana_user_pda_current
|
||||
WHERE LOWER(login)=LOWER(?)
|
||||
AND is_server=TRUE
|
||||
AND archive_head_tx_id<>''
|
||||
AND archive_head_hash<>''
|
||||
AND archive_imported=FALSE
|
||||
""";
|
||||
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
|
||||
for (String login : approvedLogins) {
|
||||
ps.setString(1, login);
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
out.add(new ArchivePublisherHead(
|
||||
rs.getString("login"),
|
||||
rs.getString("root_key"),
|
||||
rs.getString("archive_head_tx_id"),
|
||||
rs.getString("archive_head_hash"),
|
||||
rs.getBoolean("archive_imported"),
|
||||
rs.getString("archive_last_imported_tx_id")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Сдвигает локальный курсор импорта внутри FULL archive chain.
|
||||
* Update выполняется только если текущий head всё ещё тот же, с которым работает importer.
|
||||
*/
|
||||
public boolean advanceImportCursor(String publisherLogin, String expectedHeadTxId, String importedTxId) throws SQLException {
|
||||
String sql = """
|
||||
UPDATE solana_user_pda_current
|
||||
SET archive_last_imported_tx_id=?
|
||||
WHERE LOWER(login)=LOWER(?)
|
||||
AND archive_head_tx_id=?
|
||||
""";
|
||||
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
|
||||
ps.setString(1, importedTxId == null ? "" : importedTxId);
|
||||
ps.setString(2, publisherLogin);
|
||||
ps.setString(3, expectedHeadTxId);
|
||||
return ps.executeUpdate() == 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Помечает именно текущий head как полностью импортированный.
|
||||
* Если Solana users sync успел заменить head на более новый, update не произойдёт.
|
||||
*/
|
||||
public boolean markHeadImported(String publisherLogin, String expectedHeadTxId) throws SQLException {
|
||||
String sql = """
|
||||
UPDATE solana_user_pda_current
|
||||
SET archive_imported=TRUE,
|
||||
archive_last_imported_tx_id=archive_head_tx_id
|
||||
WHERE LOWER(login)=LOWER(?)
|
||||
AND archive_head_tx_id=?
|
||||
""";
|
||||
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
|
||||
ps.setString(1, publisherLogin);
|
||||
ps.setString(2, expectedHeadTxId);
|
||||
return ps.executeUpdate() == 1;
|
||||
}
|
||||
}
|
||||
|
||||
public void upsertLocation(ArchiveBlockchainLocation e) throws SQLException {
|
||||
String sql="""
|
||||
INSERT INTO archive_blockchain_location(
|
||||
blockchain_name,publisher_login,arweave_tx_id,archive_hash,big_block_number,
|
||||
chunk_offset,chunk_size,source_last_block_number,updated_at_ms)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(blockchain_name) DO UPDATE SET
|
||||
publisher_login=EXCLUDED.publisher_login,arweave_tx_id=EXCLUDED.arweave_tx_id,
|
||||
archive_hash=EXCLUDED.archive_hash,big_block_number=EXCLUDED.big_block_number,
|
||||
chunk_offset=EXCLUDED.chunk_offset,chunk_size=EXCLUDED.chunk_size,
|
||||
source_last_block_number=EXCLUDED.source_last_block_number,updated_at_ms=EXCLUDED.updated_at_ms
|
||||
WHERE EXCLUDED.source_last_block_number >= archive_blockchain_location.source_last_block_number
|
||||
""";
|
||||
try(Connection c=db.getConnection(); PreparedStatement ps=c.prepareStatement(sql)) {
|
||||
ps.setString(1,e.blockchainName()); ps.setString(2,e.publisherLogin()); ps.setString(3,e.arweaveTxId());
|
||||
ps.setBytes(4,e.archiveHash()); ps.setLong(5,e.bigBlockNumber()); ps.setLong(6,e.chunkOffset());
|
||||
ps.setLong(7,e.chunkSize()); ps.setLong(8,e.sourceLastBlockNumber()); ps.setLong(9,e.updatedAtMs()); ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
public ArchiveBlockchainLocation getLocation(String blockchainName) throws SQLException {
|
||||
String sql="""
|
||||
SELECT blockchain_name,publisher_login,arweave_tx_id,archive_hash,big_block_number,
|
||||
chunk_offset,chunk_size,source_last_block_number,updated_at_ms
|
||||
FROM archive_blockchain_location WHERE LOWER(blockchain_name)=LOWER(?)
|
||||
""";
|
||||
try(Connection c=db.getConnection(); PreparedStatement ps=c.prepareStatement(sql)) {
|
||||
ps.setString(1,blockchainName);
|
||||
try(ResultSet rs=ps.executeQuery()) {
|
||||
if(!rs.next()) return null;
|
||||
return new ArchiveBlockchainLocation(rs.getString(1),rs.getString(2),rs.getString(3),rs.getBytes(4),
|
||||
rs.getLong(5),rs.getLong(6),rs.getLong(7),rs.getLong(8),rs.getLong(9));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Данные PDA, нужные для создания пустого локального blockchain_state перед первым архивным импортом. */
|
||||
public BlockchainIdentity getBlockchainIdentity(String blockchainName) throws SQLException {
|
||||
String sql="""
|
||||
SELECT login, blockchain_name, blockchain_key, paid_limit_bytes
|
||||
FROM solana_user_pda_current WHERE LOWER(blockchain_name)=LOWER(?) LIMIT 1
|
||||
""";
|
||||
try(Connection c=db.getConnection(); PreparedStatement ps=c.prepareStatement(sql)) {
|
||||
ps.setString(1,blockchainName);
|
||||
try(ResultSet rs=ps.executeQuery()) {
|
||||
if(!rs.next()) return null;
|
||||
return new BlockchainIdentity(rs.getString(1),rs.getString(2),rs.getString(3),rs.getLong(4));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public record BlockchainIdentity(String login,String blockchainName,String blockchainKey,long sizeLimit) {}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
-- v23: trusted SHINE-ARCHIVE import state + last chunk location index.
|
||||
-- Solana archive head itself is already copied by the normal Solana users sync (v22).
|
||||
|
||||
ALTER TABLE solana_user_pda_current
|
||||
ADD COLUMN IF NOT EXISTS archive_imported BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
ALTER TABLE solana_user_pda_current
|
||||
ADD COLUMN IF NOT EXISTS archive_last_imported_tx_id TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_solana_user_pda_current_archive_pending
|
||||
ON solana_user_pda_current(is_server, archive_imported)
|
||||
WHERE archive_head_tx_id <> '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS archive_blockchain_location (
|
||||
blockchain_name TEXT PRIMARY KEY,
|
||||
publisher_login TEXT NOT NULL,
|
||||
arweave_tx_id TEXT NOT NULL,
|
||||
archive_hash BYTEA NOT NULL,
|
||||
big_block_number BIGINT NOT NULL,
|
||||
chunk_offset BIGINT NOT NULL,
|
||||
chunk_size BIGINT NOT NULL,
|
||||
source_last_block_number BIGINT NOT NULL,
|
||||
updated_at_ms BIGINT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_archive_blockchain_location_publisher
|
||||
ON archive_blockchain_location(publisher_login);
|
||||
|
||||
UPDATE db_schema_version
|
||||
SET schema_version = 23,
|
||||
updated_at_ms = CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT)
|
||||
WHERE id = 1;
|
||||
@@ -2083,8 +2083,25 @@ ALTER TABLE solana_user_pda_history
|
||||
ALTER TABLE solana_user_pda_history
|
||||
ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT '';
|
||||
|
||||
|
||||
ALTER TABLE solana_user_pda_current
|
||||
ADD COLUMN IF NOT EXISTS archive_imported BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
ALTER TABLE solana_user_pda_current
|
||||
ADD COLUMN IF NOT EXISTS archive_last_imported_tx_id TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_solana_user_pda_current_archive_pending
|
||||
ON solana_user_pda_current(is_server, archive_imported)
|
||||
WHERE archive_head_tx_id <> '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS archive_blockchain_location (
|
||||
blockchain_name TEXT PRIMARY KEY, publisher_login TEXT NOT NULL, arweave_tx_id TEXT NOT NULL,
|
||||
archive_hash BYTEA NOT NULL, big_block_number BIGINT NOT NULL, chunk_offset BIGINT NOT NULL,
|
||||
chunk_size BIGINT NOT NULL, source_last_block_number BIGINT NOT NULL, updated_at_ms BIGINT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_archive_blockchain_location_publisher ON archive_blockchain_location(publisher_login);
|
||||
|
||||
INSERT INTO db_schema_version(id,schema_version,updated_at_ms)
|
||||
VALUES(1,22,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
|
||||
VALUES(1,23,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
|
||||
ON CONFLICT(id) DO UPDATE SET schema_version=EXCLUDED.schema_version, updated_at_ms=EXCLUDED.updated_at_ms;
|
||||
|
||||
COMMIT;
|
||||
|
||||
+4
@@ -42,8 +42,10 @@ import server.logic.ws_protocol.JSON.handlers.auth.entyties.Net_UpsertEspPairing
|
||||
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_GetBlockchainBlock_Handler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_GetArchiveBlockchainLocation_Handler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.Net_AddBlock_Request;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.Net_GetBlockchainBlock_Request;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.Net_GetArchiveBlockchainLocation_Request;
|
||||
|
||||
import server.logic.ws_protocol.JSON.handlers.tempToTest.Net_GetUser_Handler;
|
||||
import server.logic.ws_protocol.JSON.handlers.tempToTest.entyties.Net_GetUser_Request;
|
||||
@@ -191,6 +193,7 @@ public final class JsonHandlerRegistry {
|
||||
// --- blockchain ---
|
||||
Map.entry("AddBlock", new Net_AddBlock_Handler()),
|
||||
Map.entry("GetBlockchainBlock", new Net_GetBlockchainBlock_Handler()),
|
||||
Map.entry("GetArchiveBlockchainLocation", new Net_GetArchiveBlockchainLocation_Handler()),
|
||||
|
||||
// --- userParams ---
|
||||
Map.entry("UpsertUserParam", new Net_UpsertUserParam_Handler()),
|
||||
@@ -283,6 +286,7 @@ public final class JsonHandlerRegistry {
|
||||
// --- blockchain ---
|
||||
Map.entry("AddBlock", Net_AddBlock_Request.class),
|
||||
Map.entry("GetBlockchainBlock", Net_GetBlockchainBlock_Request.class),
|
||||
Map.entry("GetArchiveBlockchainLocation", Net_GetArchiveBlockchainLocation_Request.class),
|
||||
|
||||
// --- userParams ---
|
||||
Map.entry("UpsertUserParam", Net_UpsertUserParam_Request.class),
|
||||
|
||||
+42
-3
@@ -102,7 +102,8 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
|
||||
blockchainName,
|
||||
req.getBlockNumber(), // старое поле, пока оставляем
|
||||
req.getPrevBlockHash(), // старое поле, пока оставляем
|
||||
req.getBlockBytesB64()
|
||||
req.getBlockBytesB64(),
|
||||
true
|
||||
);
|
||||
|
||||
// ✅ УСПЕХ: как раньше
|
||||
@@ -188,7 +189,8 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
|
||||
String blockchainName,
|
||||
int globalNumberFromReq,
|
||||
String prevGlobalHashHexFromReq,
|
||||
String blockBytesB64
|
||||
String blockBytesB64,
|
||||
boolean replicateAfterWrite
|
||||
) {
|
||||
if (blockchainName == null || blockchainName.isBlank()) {
|
||||
log.warn("AddBlock: пустой blockchainName (reqGlobalNumber={})", globalNumberFromReq);
|
||||
@@ -613,11 +615,48 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
|
||||
log.info("✅ AddBlock ok: login={}, blockchainName={}, blockNumber={}, newHash={}",
|
||||
login, blockchainName, block.blockNumber, newHashHex);
|
||||
|
||||
addBlockSyncService.replicateAsync(blockchainName, block.blockNumber);
|
||||
if (replicateAfterWrite) {
|
||||
addBlockSyncService.replicateAsync(blockchainName, block.blockNumber);
|
||||
}
|
||||
|
||||
return new AddBlockResult(WireCodes.Status.OK, null, block.blockNumber, newHashHex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Архивный импорт использует ту же проверку и запись, что обычный AddBlock,
|
||||
* но не запускает повторную межсерверную репликацию уже импортированного блока.
|
||||
*/
|
||||
public ArchiveImportResult addBlockFromArchive(String blockchainName, byte[] rawBlockBytes) {
|
||||
if (BlockchainResyncGuard.isBlockedForExternalAddBlock(blockchainName)) {
|
||||
return new ArchiveImportResult(false, "chain_resync_in_progress", -1, "");
|
||||
}
|
||||
if (rawBlockBytes == null || rawBlockBytes.length == 0) {
|
||||
return new ArchiveImportResult(false, "empty_block", -1, "");
|
||||
}
|
||||
final BchBlockEntry parsed;
|
||||
try {
|
||||
parsed = new BchBlockEntry(rawBlockBytes);
|
||||
} catch (Exception e) {
|
||||
return new ArchiveImportResult(false, "bad_block_format", -1, "");
|
||||
}
|
||||
ReentrantLock lock = BlockchainLocks.lockFor(blockchainName);
|
||||
lock.lock();
|
||||
try {
|
||||
AddBlockResult r = addBlock(
|
||||
blockchainName,
|
||||
parsed.blockNumber,
|
||||
toHex(parsed.prevHash32),
|
||||
Base64Ws.encode(rawBlockBytes),
|
||||
false
|
||||
);
|
||||
return new ArchiveImportResult(r.isOk(), r.reasonCode, r.serverLastBlockNumber, r.serverLastBlockHashHex);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public record ArchiveImportResult(boolean ok, String reasonCode, int serverLastBlockNumber, String serverLastBlockHashHex) {}
|
||||
|
||||
/* ===================================================================== */
|
||||
/* ====================== Helpers ====================================== */
|
||||
/* ===================================================================== */
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package server.logic.ws_protocol.JSON.handlers.blockchain;
|
||||
|
||||
import server.logic.ws_protocol.JSON.ConnectionContext;
|
||||
import server.logic.ws_protocol.JSON.entyties.*;
|
||||
import server.logic.ws_protocol.JSON.handlers.JsonMessageHandler;
|
||||
import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.*;
|
||||
import server.logic.ws_protocol.JSON.utils.NetExceptionResponseFactory;
|
||||
import server.logic.ws_protocol.WireCodes;
|
||||
import shine.db.archive.ArchiveBlockchainLocation;
|
||||
import shine.db.archive.ArchiveBigBlockRef;
|
||||
import shine.db.archive.ArchiveChainCursor;
|
||||
import shine.db.dao.ArchiveImportDAO;
|
||||
import shine.db.dao.ArchivePublicationDAO;
|
||||
import utils.config.AppConfig;
|
||||
|
||||
import java.util.Base64;
|
||||
|
||||
public final class Net_GetArchiveBlockchainLocation_Handler implements JsonMessageHandler {
|
||||
@Override public Net_Response handle(Net_Request base, ConnectionContext ctx) {
|
||||
Net_GetArchiveBlockchainLocation_Request req=(Net_GetArchiveBlockchainLocation_Request)base;
|
||||
String bch=String.valueOf(req.getBlockchainName()==null?"":req.getBlockchainName()).trim();
|
||||
if(bch.isEmpty()) return NetExceptionResponseFactory.error(req,WireCodes.Status.BAD_REQUEST,"BAD_FIELDS","blockchainName обязателен");
|
||||
try {
|
||||
ArchiveImportDAO importDao = ArchiveImportDAO.getInstance();
|
||||
ArchiveBlockchainLocation e=importDao.getLocation(bch);
|
||||
if (e == null) {
|
||||
// Backward-compatible fallback для publisher-а, который уже имел v22 cursors до появления v23 location index.
|
||||
ArchivePublicationDAO publicationDao = ArchivePublicationDAO.getInstance();
|
||||
ArchiveChainCursor cursor = publicationDao.getCursor(bch);
|
||||
if (cursor != null) {
|
||||
ArchiveBigBlockRef matching = publicationDao.listFinalizedBigBlocks().stream()
|
||||
.filter(ref -> ref.bigBlockNumber() == cursor.lastArchiveBigBlockNumber())
|
||||
.findFirst().orElse(null);
|
||||
if (matching != null && matching.arweaveTxId() != null && matching.arweaveTxId().length == 32) {
|
||||
String publisher = java.util.Objects.toString(AppConfig.getInstance().getParam("server.SHiNE.login"), "").trim();
|
||||
String txId = Base64.getUrlEncoder().withoutPadding().encodeToString(matching.arweaveTxId());
|
||||
e = new ArchiveBlockchainLocation(
|
||||
bch, publisher, txId, cursor.lastArchiveBigBlockHash(), cursor.lastArchiveBigBlockNumber(),
|
||||
cursor.lastChunkOffset(), cursor.lastChunkSize(), cursor.lastArchivedSourceBlockNumber(), System.currentTimeMillis());
|
||||
importDao.upsertLocation(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
if(e==null) return NetExceptionResponseFactory.error(req,WireCodes.Status.NOT_FOUND,"ARCHIVE_LOCATION_NOT_FOUND","Архивная ссылка пока не известна этому серверу");
|
||||
Net_GetArchiveBlockchainLocation_Response r=new Net_GetArchiveBlockchainLocation_Response();
|
||||
r.setOp(req.getOp()); r.setRequestId(req.getRequestId()); r.setStatus(WireCodes.Status.OK);
|
||||
r.setBlockchainName(e.blockchainName()); r.setPublisherLogin(e.publisherLogin()); r.setArweaveTxId(e.arweaveTxId()); r.setArchiveHash(hex(e.archiveHash()));
|
||||
r.setBigBlockNumber(e.bigBlockNumber()); r.setChunkOffset(e.chunkOffset()); r.setChunkSize(e.chunkSize()); r.setSourceLastBlockNumber(e.sourceLastBlockNumber()); return r;
|
||||
} catch(Exception e) { return NetExceptionResponseFactory.error(req,WireCodes.Status.INTERNAL_ERROR,"INTERNAL_ERROR",NetExceptionResponseFactory.detailedMessage("Не удалось получить архивную ссылку",e)); }
|
||||
}
|
||||
private static String hex(byte[] b){if(b==null)return"";StringBuilder s=new StringBuilder(b.length*2);for(byte x:b)s.append(String.format("%02x",x));return s.toString();}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package server.logic.ws_protocol.JSON.handlers.blockchain.entyties;
|
||||
|
||||
import server.logic.ws_protocol.JSON.entyties.Net_Request;
|
||||
|
||||
public final class Net_GetArchiveBlockchainLocation_Request extends Net_Request {
|
||||
private String blockchainName;
|
||||
public String getBlockchainName(){return blockchainName;}
|
||||
public void setBlockchainName(String value){this.blockchainName=value;}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package server.logic.ws_protocol.JSON.handlers.blockchain.entyties;
|
||||
|
||||
import server.logic.ws_protocol.JSON.entyties.Net_Response;
|
||||
|
||||
public final class Net_GetArchiveBlockchainLocation_Response extends Net_Response {
|
||||
private String blockchainName,publisherLogin,arweaveTxId,archiveHash;
|
||||
private long bigBlockNumber,chunkOffset,chunkSize,sourceLastBlockNumber;
|
||||
public String getBlockchainName(){return blockchainName;} public void setBlockchainName(String v){blockchainName=v;}
|
||||
public String getPublisherLogin(){return publisherLogin;} public void setPublisherLogin(String v){publisherLogin=v;}
|
||||
public String getArweaveTxId(){return arweaveTxId;} public void setArweaveTxId(String v){arweaveTxId=v;}
|
||||
public String getArchiveHash(){return archiveHash;} public void setArchiveHash(String v){archiveHash=v;}
|
||||
public long getBigBlockNumber(){return bigBlockNumber;} public void setBigBlockNumber(long v){bigBlockNumber=v;}
|
||||
public long getChunkOffset(){return chunkOffset;} public void setChunkOffset(long v){chunkOffset=v;}
|
||||
public long getChunkSize(){return chunkSize;} public void setChunkSize(long v){chunkSize=v;}
|
||||
public long getSourceLastBlockNumber(){return sourceLastBlockNumber;} public void setSourceLastBlockNumber(long v){sourceLastBlockNumber=v;}
|
||||
}
|
||||
+1
-1
@@ -629,7 +629,7 @@ public final class SolanaUsersSyncService
|
||||
account.address(),
|
||||
account.slot(),
|
||||
account.dataBase64(),
|
||||
state.lastSeenSignature()
|
||||
""
|
||||
)
|
||||
);
|
||||
} catch (Exception ignored) {
|
||||
|
||||
+9
-7
@@ -299,13 +299,7 @@ public final class PostgresStorageRepository
|
||||
|
||||
connection.setAutoCommit(false);
|
||||
|
||||
try (Statement statement =
|
||||
connection.createStatement()) {
|
||||
|
||||
statement.executeUpdate(
|
||||
"TRUNCATE TABLE solana_user_pda_current"
|
||||
);
|
||||
|
||||
try {
|
||||
upsertCurrentSnapshots(
|
||||
connection,
|
||||
snapshots
|
||||
@@ -494,6 +488,14 @@ public final class PostgresStorageRepository
|
||||
"last_block_hash = EXCLUDED.last_block_hash, " +
|
||||
"last_block_signature = EXCLUDED.last_block_signature, " +
|
||||
"arweave_tx_id = EXCLUDED.arweave_tx_id, " +
|
||||
"archive_imported = CASE WHEN " +
|
||||
"EXCLUDED.archive_head_tx_id IS DISTINCT FROM solana_user_pda_current.archive_head_tx_id " +
|
||||
"OR EXCLUDED.archive_head_hash IS DISTINCT FROM solana_user_pda_current.archive_head_hash " +
|
||||
"THEN FALSE ELSE solana_user_pda_current.archive_imported END, " +
|
||||
"archive_last_imported_tx_id = CASE WHEN " +
|
||||
"EXCLUDED.archive_head_tx_id = solana_user_pda_current.archive_head_tx_id " +
|
||||
"AND EXCLUDED.archive_head_hash IS DISTINCT FROM solana_user_pda_current.archive_head_hash " +
|
||||
"THEN '' ELSE solana_user_pda_current.archive_last_imported_tx_id END, " +
|
||||
"archive_head_tx_id = EXCLUDED.archive_head_tx_id, " +
|
||||
"archive_head_hash = EXCLUDED.archive_head_hash, " +
|
||||
"is_server = EXCLUDED.is_server, " +
|
||||
|
||||
@@ -6,6 +6,7 @@ import org.eclipse.jetty.websocket.server.config.JettyWebSocketServletContainerI
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import server.archive.ArchivePublisherScheduler;
|
||||
import server.archive.ArchiveImportScheduler;
|
||||
import server.debug.DebugApiConfigurator;
|
||||
import server.files.DmFileApiConfigurator;
|
||||
import server.sync.BlockchainResyncRecoveryOnStartup;
|
||||
@@ -85,6 +86,8 @@ public final class WsServer {
|
||||
|
||||
// Опциональная серверная публикация больших SHINE-ARCHIVE блоков.
|
||||
ArchivePublisherScheduler.startOrLog();
|
||||
// Импорт больших архивов только от publisher-ов из archive.import.allowedPublishers.
|
||||
ArchiveImportScheduler.startOrLog();
|
||||
|
||||
// ============================================================
|
||||
// 2) Запуск Jetty WS
|
||||
@@ -114,7 +117,7 @@ public final class WsServer {
|
||||
log.info("✅ WS сервер запущен на ws://localhost:{}/ws", port);
|
||||
ServerConnectionPool.getInstance().startOrLog();
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(
|
||||
() -> { ServerConnectionPool.getInstance().close(); ArchivePublisherScheduler.close(); },
|
||||
() -> { ServerConnectionPool.getInstance().close(); ArchivePublisherScheduler.close(); ArchiveImportScheduler.close(); },
|
||||
"server-connection-pool-shutdown"));
|
||||
PeriodicDmDeliveryService.startOrLog();
|
||||
server.join();
|
||||
|
||||
@@ -155,3 +155,9 @@ test.freeAvatar.limitPerUser=3
|
||||
test.freeAvatar.maxBytes=131072
|
||||
test.freeAvatar.walletAddress=
|
||||
test.freeAvatar.walletJwkPath=
|
||||
|
||||
# Trusted SHINE-ARCHIVE import. Empty list = do not trust/import archives from anyone.
|
||||
# Comma-separated SHiNE server logins, case-insensitive.
|
||||
archive.import.allowedPublishers=
|
||||
archive.import.intervalMinutes=60
|
||||
archive.import.workDir=data/archive-import
|
||||
|
||||
Reference in New Issue
Block a user