Архивация в Arweave

This commit is contained in:
AidarKC
2026-09-12 10:17:23 +03:00
parent ef068eca81
commit c3253dceac
48 changed files with 6681 additions and 1227 deletions
@@ -0,0 +1,31 @@
plugins {
id 'java'
}
group = 'shine'
version = '1.0.0'
java {
toolchain {
languageVersion = JavaLanguageVersion.of(17)
}
}
repositories { mavenCentral() }
dependencies {
implementation project(':shine-server-config')
implementation project(':shine-server-db')
implementation project(':shine-server-crypto')
implementation project(':shine-server-blockchain')
implementation project(':shine-server-net-protocol')
implementation project(':shine-server-solana-users-sync')
implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1'
implementation 'org.slf4j:slf4j-api:2.0.16'
testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
}
test { useJUnitPlatform() }
@@ -0,0 +1,56 @@
package server.archive;
import java.nio.file.Path;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
/** Единые локальные имена больших архивных файлов. */
public final class ArchiveFileNames {
private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("dd.MM.yy");
private ArchiveFileNames() {}
/**
* Временное имя полностью собранного файла до появления Arweave TX ID.
* Пример: archive01.00001.11.09.26.tmp.SHiNE-archive
*/
public static String pendingFileName(String login, long bigBlockNumber, long createdAtMs, ZoneId zone) {
return baseName(login, bigBlockNumber, createdAtMs, zone) + ".tmp.SHiNE-archive";
}
/**
* Постоянное имя после успешной загрузки в Arweave.
* Пример: archive01.00001.11.09.26.Xm32...kP9.SHiNE-archive
*/
public static String uploadedFileName(String login, long bigBlockNumber, long createdAtMs, ZoneId zone, byte[] txId) {
if (txId == null || txId.length != 32) throw new IllegalArgumentException("Arweave TX ID должен быть 32 bytes");
String tx = Base64.getUrlEncoder().withoutPadding().encodeToString(txId);
return baseName(login, bigBlockNumber, createdAtMs, zone) + "." + tx + ".SHiNE-archive";
}
public static Path pendingFilePath(Path archiveDir, String login, long bigBlockNumber, long createdAtMs, ZoneId zone) {
return archiveDir.resolve(pendingFileName(login, bigBlockNumber, createdAtMs, zone));
}
public static Path uploadedFilePath(Path archiveDir, String login, long bigBlockNumber, long createdAtMs, ZoneId zone, byte[] txId) {
return archiveDir.resolve(uploadedFileName(login, bigBlockNumber, createdAtMs, zone, txId));
}
private static String baseName(String login, long bigBlockNumber, long createdAtMs, ZoneId zone) {
if (bigBlockNumber < 0 || bigBlockNumber > 0xffff_ffffL) {
throw new IllegalArgumentException("Номер большого блока вне u32: " + bigBlockNumber);
}
String safeLogin = sanitize(login);
String number = String.format("%05d", bigBlockNumber);
String date = DATE.format(Instant.ofEpochMilli(createdAtMs).atZone(zone == null ? ZoneId.systemDefault() : zone));
return safeLogin + "." + number + "." + date;
}
static String sanitize(String login) {
String s = login == null ? "server" : login.trim();
if (s.isEmpty()) s = "server";
return s.replaceAll("[^A-Za-z0-9._-]", "_");
}
}
@@ -0,0 +1,75 @@
package server.archive;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import sync.util.Base58Util;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
/** Читает Ed25519 seed32 из распространённых SHiNE/Solana форматов ключей. */
final class ArchiveKeyLoader {
private static final ObjectMapper JSON = new ObjectMapper();
private ArchiveKeyLoader() {}
static byte[] loadSeed32(Path path) throws IOException {
if (path == null) throw new IllegalArgumentException("Путь ключа не задан");
byte[] raw = Files.readAllBytes(path);
if (raw.length == 32) return raw.clone();
if (raw.length == 64) return first32(raw);
String text = new String(raw, StandardCharsets.UTF_8).trim();
if (text.isEmpty()) throw new IOException("Пустой key file: " + path);
// Solana keypair JSON: [64 чисел].
if (text.startsWith("[")) {
JsonNode arr = JSON.readTree(text);
if (!arr.isArray() || (arr.size() != 32 && arr.size() != 64)) {
throw new IOException("Ожидался JSON keypair на 32/64 байта: " + path);
}
byte[] bytes = new byte[arr.size()];
for (int i = 0; i < arr.size(); i++) {
int v = arr.get(i).asInt(-1);
if (v < 0 || v > 255) throw new IOException("Некорректный байт keypair: " + path);
bytes[i] = (byte) v;
}
return bytes.length == 32 ? bytes : first32(bytes);
}
// Base58 seed32 или Solana secret key 64 bytes.
try {
byte[] decoded = Base58Util.decode(text);
if (decoded.length == 32) return decoded;
if (decoded.length == 64) return first32(decoded);
} catch (IllegalArgumentException ignored) {
// Пробуем остальные текстовые форматы ниже.
}
// Base64 raw seed / keypair / Ed25519 PKCS8. В SHiNE PKCS8 seed находится в последних 32 байтах.
try {
byte[] decoded = Base64.getDecoder().decode(text.replaceAll("\\s+", ""));
if (decoded.length == 32) return decoded;
if (decoded.length == 64) return first32(decoded);
if (decoded.length > 32) {
byte[] seed = new byte[32];
System.arraycopy(decoded, decoded.length - 32, seed, 0, 32);
return seed;
}
} catch (IllegalArgumentException ignored) {
// Ниже выдадим понятную общую ошибку.
}
throw new IOException("Не удалось распознать Ed25519 key file: " + path
+ ". Поддерживаются raw seed32, raw/keypair64, Solana JSON, Base58 seed/keypair и Base64 PKCS8.");
}
private static byte[] first32(byte[] bytes) {
byte[] out = new byte[32];
System.arraycopy(bytes, 0, out, 0, 32);
return out;
}
}
@@ -0,0 +1,98 @@
package server.archive;
import utils.config.AppConfig;
import utils.config.SolanaProgramsConfig;
import java.nio.file.Path;
import java.time.LocalTime;
import java.time.ZoneId;
/** Настройки опционального архивного publisher-а. По умолчанию функция выключена. */
public record ArchivePublisherConfig(
boolean enabled,
LocalTime publishTime,
ZoneId publishZone,
long maxFileBytes,
Path workDir,
String serverLogin,
String arweaveGateway,
Path arweaveWalletJwkPath,
int arweaveMinConfirmations,
int arweaveConfirmPollSeconds,
int arweaveConfirmTimeoutMinutes,
Path solanaRootKeyPath,
Path solanaClientKeyPath,
String solanaRpcUrl,
String solanaUsersProgramId,
String solanaPaymentsProgramId,
int solanaConfirmPollSeconds,
int solanaConfirmTimeoutMinutes,
String solanaCommitment
) {
public static ArchivePublisherConfig load() {
AppConfig c = AppConfig.getInstance();
return new ArchivePublisherConfig(
c.getBoolean("archive.publish.enabled", false),
parseTime(orDefault(c.getParam("archive.publish.time"), "00:00")),
parseZone(c.getParam("archive.publish.zoneId")),
positiveLong(parseLong(c.getParam("archive.maxFileBytes"), 4_000_000_000L), "archive.maxFileBytes"),
Path.of(orDefault(c.getParam("archive.workDir"), "data/archive")),
requiredIfEnabled(c, "server.SHiNE.login"),
orDefault(c.getParam("archive.arweave.gateway"), "https://arweave.net"),
optionalPath(c.getParam("archive.arweave.walletJwkPath")),
nonNegative(c.getInt("archive.arweave.minConfirmations", 1), "archive.arweave.minConfirmations"),
positive(c.getInt("archive.arweave.confirmPollSeconds", 30), "archive.arweave.confirmPollSeconds"),
positive(c.getInt("archive.arweave.confirmTimeoutMinutes", 180), "archive.arweave.confirmTimeoutMinutes"),
optionalPath(c.getParam("archive.solana.rootKeyPath")),
optionalPath(c.getParam("archive.solana.clientKeyPath")),
firstNonBlank(c.getParam("solana.users.sync.rpcUrl"), c.getParam("solana.rpcUrl"), SolanaProgramsConfig.SOLANA_RPC_URL),
firstNonBlank(c.getParam("solana.users.sync.programId"), SolanaProgramsConfig.SHINE_USERS_PROGRAM_ID),
SolanaProgramsConfig.SHINE_PAYMENTS_PROGRAM_ID,
positive(c.getInt("archive.solana.confirmPollSeconds", 5), "archive.solana.confirmPollSeconds"),
positive(c.getInt("archive.solana.confirmTimeoutMinutes", 30), "archive.solana.confirmTimeoutMinutes"),
orDefault(c.getParam("archive.solana.commitment"), "finalized")
);
}
public void validateForStart() {
if (!enabled) return;
if (serverLogin == null || serverLogin.isBlank()) throw new IllegalArgumentException("server.SHiNE.login обязателен для archive publisher");
if (arweaveWalletJwkPath == null) throw new IllegalArgumentException("archive.arweave.walletJwkPath обязателен");
if (solanaRootKeyPath == null) throw new IllegalArgumentException("archive.solana.rootKeyPath обязателен");
if (solanaClientKeyPath == null) throw new IllegalArgumentException("archive.solana.clientKeyPath обязателен");
if (solanaRpcUrl == null || solanaRpcUrl.isBlank()) throw new IllegalArgumentException("solana.users.sync.rpcUrl или solana.rpcUrl обязателен");
if (maxFileBytes >= 0x1_0000_0000L) {
throw new IllegalArgumentException("SHINE-ARCHIVE v1 требует archive.maxFileBytes < 4294967296");
}
}
private static String requiredIfEnabled(AppConfig c, String name) {
String v = c.getParam(name);
return v == null ? "" : v.trim();
}
private static Path optionalPath(String s) { return s == null || s.isBlank() ? null : Path.of(s.trim()); }
private static String firstNonBlank(String... values) {
if (values != null) for (String v : values) if (v != null && !v.isBlank()) return v.trim();
return "";
}
private static String orDefault(String v, String d) { return v == null || v.isBlank() ? d : v.trim(); }
private static long parseLong(String s, long d) { return s == null || s.isBlank() ? d : Long.parseLong(s.trim()); }
private static LocalTime parseTime(String value) {
try {
return LocalTime.parse(value.trim());
} catch (Exception e) {
throw new IllegalArgumentException("archive.publish.time должен быть в формате HH:mm, например 00:00", e);
}
}
private static ZoneId parseZone(String value) {
if (value == null || value.isBlank()) return ZoneId.systemDefault();
try {
return ZoneId.of(value.trim());
} catch (Exception e) {
throw new IllegalArgumentException("Некорректный archive.publish.zoneId: " + value, e);
}
}
private static int positive(int v, String n) { if (v <= 0) throw new IllegalArgumentException(n + " должен быть > 0"); return v; }
private static int nonNegative(int v, String n) { if (v < 0) throw new IllegalArgumentException(n + " должен быть >= 0"); return v; }
private static long positiveLong(long v, String n) { if (v <= 0) throw new IllegalArgumentException(n + " должен быть > 0"); return v; }
}
@@ -0,0 +1,90 @@
package server.archive;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/** Опциональный scheduler: один новый archive job в заданное локальное время каждый день. */
public final class ArchivePublisherScheduler {
private static final Logger log = LoggerFactory.getLogger(ArchivePublisherScheduler.class);
private static final AtomicBoolean STARTED = new AtomicBoolean(false);
private static ScheduledExecutorService executor;
private ArchivePublisherScheduler() {}
public static void startOrLog() {
ArchivePublisherConfig cfg;
try {
cfg = ArchivePublisherConfig.load();
} catch (Exception e) {
log.error("Не удалось прочитать archive publisher config", e);
return;
}
if (!cfg.enabled()) {
log.info("Archive publisher выключен (archive.publish.enabled=false)");
return;
}
if (!STARTED.compareAndSet(false, true)) return;
try {
ArchivePublisherService service = new ArchivePublisherService(cfg);
executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override public Thread newThread(Runnable r) {
Thread t = new Thread(r, "shine-archive-publisher");
t.setDaemon(true);
return t;
}
});
// Незавершённую публикацию после рестарта продолжаем сразу, а новый snapshot
// создаём только в очередное назначенное суточное время.
executor.execute(() -> {
try {
service.resumeUnfinishedJob();
} catch (Exception e) {
log.error("Не удалось продолжить незавершённый archive job", e);
}
});
scheduleNextDaily(service, cfg);
log.info("Archive publisher включён: login={} dir={} dailyAt={} zone={}",
cfg.serverLogin(), cfg.workDir(), cfg.publishTime(), cfg.publishZone());
} catch (Exception e) {
STARTED.set(false);
log.error("Archive publisher НЕ запущен из-за ошибки preflight", e);
}
}
private static void scheduleNextDaily(ArchivePublisherService service, ArchivePublisherConfig cfg) {
ScheduledExecutorService e = executor;
if (e == null || e.isShutdown()) return;
ZonedDateTime now = ZonedDateTime.now(cfg.publishZone());
ZonedDateTime next = now.toLocalDate().atTime(cfg.publishTime()).atZone(cfg.publishZone());
if (!next.isAfter(now)) next = next.plusDays(1);
long delayMs = Math.max(1L, Duration.between(now, next).toMillis());
e.schedule(() -> {
try {
service.runCycle();
} catch (Exception ex) {
log.error("Archive publisher daily cycle failed; retry нового snapshot будет в следующее назначенное время", ex);
} finally {
scheduleNextDaily(service, cfg);
}
}, delayMs, TimeUnit.MILLISECONDS);
log.info("Следующая архивная публикация запланирована на {}", next);
}
public static void close() {
ScheduledExecutorService e = executor;
if (e != null) e.shutdownNow();
}
}
@@ -0,0 +1,302 @@
package server.archive;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import server.logic.ws_protocol.JSON.handlers.blockchain.Net_AddBlock_Handler_utils.BlockchainLocks;
import shine.db.archive.ArchiveBigBlockRef;
import shine.db.archive.ArchiveChainCursor;
import shine.db.archive.ArchivePublishJob;
import shine.db.archive.ArchivePublishJobChain;
import shine.db.dao.ArchivePublicationDAO;
import shine.db.dao.BlockchainStateDAO;
import shine.db.dao.BlocksDAO;
import shine.db.entities.BlockEntry;
import shine.db.entities.BlockchainStateEntry;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
/** Один crash-safe цикл публикации большого SHINE-ARCHIVE блока. */
public final class ArchivePublisherService {
private static final Logger log = LoggerFactory.getLogger(ArchivePublisherService.class);
private final ArchivePublisherConfig cfg;
private final ArchivePublicationDAO archiveDao = ArchivePublicationDAO.getInstance();
private final BlockchainStateDAO stateDao = BlockchainStateDAO.getInstance();
private final BlocksDAO blocksDao = BlocksDAO.getInstance();
private final ShineArchiveWriter writer = new ShineArchiveWriter();
private final ArweaveArchiveService arweave;
private final SolanaArchiveHeadWriter solana;
private final byte[] archiveSigningSeed32;
private final AtomicBoolean running = new AtomicBoolean(false);
public ArchivePublisherService(ArchivePublisherConfig cfg) throws Exception {
this.cfg = Objects.requireNonNull(cfg);
cfg.validateForStart();
Files.createDirectories(cfg.workDir());
this.archiveSigningSeed32 = ArchiveKeyLoader.loadSeed32(cfg.solanaRootKeyPath());
this.arweave = new ArweaveArchiveService(cfg);
this.solana = new SolanaArchiveHeadWriter(cfg);
this.solana.validateKeysAgainstPda();
}
/** После рестарта продолжает только уже существующий незавершённый job; новый snapshot не создаёт. */
public boolean resumeUnfinishedJob() throws Exception {
if (!running.compareAndSet(false, true)) return false;
try {
ArchivePublishJob job = archiveDao.findUnfinishedJob();
if (job == null) return false;
log.info("Archive publisher: после рестарта продолжаем job id={} block={} status={}",
job.id(), job.bigBlockNumber(), job.status());
process(job.id());
return true;
} finally {
running.set(false);
}
}
/** Запускает суточный цикл: продолжает незавершённый job либо создаёт новый frozen snapshot. */
public boolean runCycle() throws Exception {
if (!running.compareAndSet(false, true)) {
log.info("Archive publisher: предыдущий цикл ещё работает, новый пропущен");
return false;
}
try {
ArchivePublishJob job = archiveDao.findUnfinishedJob();
if (job == null) {
job = createFrozenJob();
if (job == null) {
log.info("Archive publisher: новых SHiNE-блоков нет");
return false;
}
} else {
log.info("Archive publisher: продолжаем незавершённый job id={} block={} status={}",
job.id(), job.bigBlockNumber(), job.status());
}
process(job.id());
return true;
} finally {
running.set(false);
}
}
private ArchivePublishJob createFrozenJob() throws Exception {
List<ArchivePublishJobChain> chains = new ArrayList<>();
for (BlockchainStateEntry listed : stateDao.listAll()) {
if (listed == null || listed.getBlockchainName() == null || listed.getBlockchainName().isBlank()) continue;
String name = listed.getBlockchainName();
ReentrantLock lock = BlockchainLocks.lockFor(name);
lock.lock();
try {
BlockchainStateEntry state = stateDao.getByBlockchainName(name);
if (state == null || state.getLastBlockNumber() < 0) continue;
ArchiveChainCursor cursor = archiveDao.getCursor(name);
long from;
byte[] previousSourceHash;
Long prevBig = null;
byte[] prevBigHash = null;
Long prevChunkOffset = null;
Long prevChunkSize = null;
if (cursor == null) {
from = 0L;
previousSourceHash = new byte[32];
} else {
long cursorBlock = cursor.lastArchivedSourceBlockNumber();
if (cursorBlock < 0 || cursorBlock > 0xffff_ffffL) {
throw new IllegalStateException("Некорректный archive cursor для " + name + ": " + cursorBlock);
}
byte[] actualCursorHash = blocksDao.getHashByNumber(name, (int) cursorBlock);
if (!Arrays.equals(actualCursorHash, cursor.lastArchivedSourceBlockHash())) {
throw new IllegalStateException("Archive cursor hash не совпадает с локальной blockchain: " + name + "#" + cursorBlock);
}
from = cursorBlock + 1L;
previousSourceHash = cursor.lastArchivedSourceBlockHash();
prevBig = cursor.lastArchiveBigBlockNumber();
prevBigHash = cursor.lastArchiveBigBlockHash();
prevChunkOffset = cursor.lastChunkOffset();
prevChunkSize = cursor.lastChunkSize();
}
long to = Integer.toUnsignedLong(state.getLastBlockNumber());
if (from > to) continue;
if (from > 0xffff_ffffL || to > 0xffff_ffffL) throw new IllegalStateException("Source block number вне u32: " + name);
byte[] lastHash = state.getLastBlockHash();
if (lastHash == null || lastHash.length != 32) {
lastHash = blocksDao.getHashByNumber(name, (int) to);
}
require32(lastHash, "last source block hash " + name);
chains.add(new ArchivePublishJobChain(
0L, name, from, to, previousSourceHash, lastHash,
prevBig, prevBigHash, prevChunkOffset, prevChunkSize, null, null));
} finally {
lock.unlock();
}
}
if (chains.isEmpty()) return null;
chains.sort(Comparator.comparing(ArchivePublishJobChain::blockchainName));
long bigBlock = archiveDao.nextBigBlockNumber();
if (bigBlock > 0xffff_ffffL) throw new IllegalStateException("Закончился диапазон u32 big_block_number");
long createdAt = System.currentTimeMillis();
long id = archiveDao.createJobWithChains(bigBlock, createdAt, chains);
log.info("Archive snapshot создан: job={} bigBlock={} chains={}", id, bigBlock, chains.size());
return archiveDao.getJob(id);
}
private void process(long jobId) throws Exception {
while (true) {
ArchivePublishJob job = archiveDao.getJob(jobId);
if (job == null) throw new IllegalStateException("Archive job исчез: " + jobId);
switch (job.status()) {
case "SNAPSHOT_CREATED" -> buildFile(job);
case "FILE_BUILT" -> uploadArweave(job);
case "ARWEAVE_UPLOADED" -> waitArweave(job);
case "ARWEAVE_CONFIRMED" -> submitSolana(job);
case "SOLANA_SUBMITTED" -> waitSolana(job);
case "SOLANA_FINALIZED" -> commitCursors(job);
case "CURSORS_COMMITTED" -> {
log.info("Archive job завершён: id={} bigBlock={} file={} tx={}", job.id(), job.bigBlockNumber(), job.localArchivePath(), b64url(job.arweaveTxId()));
return;
}
case "FAILED" -> throw new IllegalStateException("Archive job помечен FAILED: " + job.errorText());
default -> throw new IllegalStateException("Неизвестный archive job status: " + job.status());
}
}
}
private void buildFile(ArchivePublishJob job) throws Exception {
List<ArchivePublishJobChain> chains = archiveDao.listJobChains(job.id());
if (chains.isEmpty()) throw new IllegalStateException("SNAPSHOT_CREATED job не содержит chains: " + job.id());
Map<String,List<BlockEntry>> records = new LinkedHashMap<>();
for (ArchivePublishJobChain ch : chains) {
ReentrantLock lock = BlockchainLocks.lockFor(ch.blockchainName());
lock.lock();
try {
if (ch.fromSourceBlockNumber() > 0) {
byte[] prev = blocksDao.getHashByNumber(ch.blockchainName(), (int)(ch.fromSourceBlockNumber() - 1));
if (!Arrays.equals(prev, ch.previousSourceBlockHash())) {
throw new IllegalStateException("Frozen previous source hash изменился: " + ch.blockchainName());
}
}
List<BlockEntry> range = blocksDao.listRangeByNumber(
ch.blockchainName(), (int)ch.fromSourceBlockNumber(), (int)ch.toSourceBlockNumber());
if (range.isEmpty()) throw new IllegalStateException("Frozen range пуст: " + ch.blockchainName());
byte[] last = range.get(range.size()-1).getBlockHash();
if (!Arrays.equals(last, ch.lastSourceBlockHash())) {
throw new IllegalStateException("Frozen last source hash изменился: " + ch.blockchainName());
}
records.put(ch.blockchainName(), range);
} finally {
lock.unlock();
}
}
List<ArchiveBigBlockRef> previous = archiveDao.listFinalizedBigBlocks();
Path file = ArchiveFileNames.pendingFilePath(
cfg.workDir(), cfg.serverLogin(), job.bigBlockNumber(), job.createdAtMs(), cfg.publishZone());
ShineArchiveWriter.BuildResult result = writer.build(
file, cfg.serverLogin(), job.bigBlockNumber(), job.createdAtMs(), previous,
chains, records, archiveSigningSeed32, cfg.maxFileBytes());
for (Map.Entry<String, ShineArchiveWriter.ChunkPosition> e : result.chunks().entrySet()) {
archiveDao.updateNewChunkPosition(job.id(), e.getKey(), e.getValue().offset(), e.getValue().size());
}
archiveDao.markFileBuilt(job.id(), result.file().toString(), result.archiveHash());
log.info("SHINE-ARCHIVE локально сохранён: block={} file={} bytes={} hash={}",
job.bigBlockNumber(), result.file(), result.fileSize(), hex(result.archiveHash()));
}
private void uploadArweave(ArchivePublishJob job) throws Exception {
Path file = requireLocalFile(job);
byte[] verified = ShineArchiveWriter.verifyLocalFile(file);
if (!Arrays.equals(verified, job.archiveHash())) throw new IllegalStateException("Локальный archive hash отличается от job DB");
ArweaveArchiveService.UploadResult upload = arweave.upload(file, cfg.serverLogin(), job.bigBlockNumber(), job.archiveHash());
// Сначала надёжно фиксируем полученный TX ID в БД. Если процесс упадёт до rename,
// recovery на следующем шаге переименует тот же временный файл без повторной загрузки.
archiveDao.markArweaveUploaded(job.id(), upload.txIdBytes());
Path uploadedFile = markLocalFileUploaded(job, file, upload.txIdBytes());
log.info("SHINE-ARCHIVE загружен в Arweave: block={} tx={} file={}",
job.bigBlockNumber(), upload.txId(), uploadedFile);
}
private void waitArweave(ArchivePublishJob job) throws Exception {
ensureUploadedLocalName(job);
String tx = b64url(job.arweaveTxId());
int confirmations = arweave.waitForConfirmations(tx);
archiveDao.markArweaveConfirmed(job.id(), confirmations);
log.info("Arweave archive закреплён: block={} tx={} confirmations={}", job.bigBlockNumber(), tx, confirmations);
}
/** После успешного Arweave upload временный файл получает реальный TX ID в имени и остаётся локально. */
private Path markLocalFileUploaded(ArchivePublishJob job, Path currentFile, byte[] txId) throws Exception {
require32(txId, "arweave tx id");
Path target = ArchiveFileNames.uploadedFilePath(
cfg.workDir(), cfg.serverLogin(), job.bigBlockNumber(), job.createdAtMs(), cfg.publishZone(), txId);
Files.createDirectories(target.toAbsolutePath().getParent());
if (!currentFile.equals(target)) {
try {
Files.move(currentFile, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(currentFile, target, StandardCopyOption.REPLACE_EXISTING);
}
}
archiveDao.updateLocalArchivePath(job.id(), target.toString());
return target;
}
/** Crash recovery: TX уже сохранён, а временный файл ещё не успели переименовать. */
private void ensureUploadedLocalName(ArchivePublishJob job) throws Exception {
require32(job.arweaveTxId(), "job.arweave_tx_id");
Path target = ArchiveFileNames.uploadedFilePath(
cfg.workDir(), cfg.serverLogin(), job.bigBlockNumber(), job.createdAtMs(), cfg.publishZone(), job.arweaveTxId());
if (Files.isRegularFile(target)) {
if (!target.toString().equals(job.localArchivePath())) {
archiveDao.updateLocalArchivePath(job.id(), target.toString());
}
return;
}
if (job.localArchivePath() == null || job.localArchivePath().isBlank()) return;
Path current = Path.of(job.localArchivePath());
if (Files.isRegularFile(current)) {
markLocalFileUploaded(job, current, job.arweaveTxId());
}
}
private void submitSolana(ArchivePublishJob job) throws Exception {
require32(job.arweaveTxId(), "job.arweave_tx_id");
require32(job.archiveHash(), "job.archive_hash");
SolanaArchiveHeadWriter.SubmitResult submitted = solana.submitArchiveHead(job.arweaveTxId(), job.archiveHash());
archiveDao.markSolanaSubmitted(job.id(), submitted.signature());
}
private void waitSolana(ArchivePublishJob job) throws Exception {
if (job.solanaSignature() == null || job.solanaSignature().isBlank()) throw new IllegalStateException("SOLANA_SUBMITTED без signature");
solana.waitFinalized(job.solanaSignature());
if (!solana.currentHeadMatches(job.arweaveTxId(), job.archiveHash())) {
throw new IllegalStateException("После Solana finalized User PDA archive head не совпадает с job");
}
archiveDao.markSolanaFinalized(job.id());
log.info("Solana archive head finalized: block={} signature={}", job.bigBlockNumber(), job.solanaSignature());
}
private void commitCursors(ArchivePublishJob job) throws Exception {
archiveDao.commitCursors(job.id(), job.archiveHash());
}
private static Path requireLocalFile(ArchivePublishJob job) {
if (job.localArchivePath() == null || job.localArchivePath().isBlank()) throw new IllegalStateException("FILE_BUILT без local_archive_path");
Path p = Path.of(job.localArchivePath());
if (!Files.isRegularFile(p)) throw new IllegalStateException("Локальный archive-файл отсутствует: " + p);
return p;
}
private static void require32(byte[] v, String name) {
if (v == null || v.length != 32) throw new IllegalStateException(name + " должен быть 32 bytes");
}
private static String b64url(byte[] v) { return v == null ? "" : Base64.getUrlEncoder().withoutPadding().encodeToString(v); }
private static String hex(byte[] b){StringBuilder s=new StringBuilder();for(byte x:b)s.append(String.format("%02x",x));return s.toString();}
}
@@ -0,0 +1,259 @@
package server.archive;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.math.BigInteger;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.*;
import java.security.spec.MGF1ParameterSpec;
import java.security.spec.PSSParameterSpec;
import java.security.spec.RSAPrivateCrtKeySpec;
import java.time.Duration;
import java.util.*;
/**
* Публикация готовых локальных SHINE-ARCHIVE файлов в Arweave v2.
* Поддерживает стандартный data Merkle tree и загрузку через /chunk.
*/
public final class ArweaveArchiveService {
private static final Logger log = LoggerFactory.getLogger(ArweaveArchiveService.class);
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final Base64.Encoder B64 = Base64.getUrlEncoder().withoutPadding();
private static final Base64.Decoder B64D = Base64.getUrlDecoder();
private final ArchivePublisherConfig cfg;
private final HttpClient http;
private volatile Wallet wallet;
public ArweaveArchiveService(ArchivePublisherConfig cfg) {
this.cfg = Objects.requireNonNull(cfg);
this.http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(20)).build();
}
public record UploadResult(String txId, byte[] txIdBytes, String gateway) {}
public UploadResult upload(Path archiveFile, String publisherLogin, long bigBlockNumber, byte[] archiveHash)
throws Exception {
long dataSize = Files.size(archiveFile);
if (dataSize <= 0) throw new IllegalArgumentException("Пустой SHINE-ARCHIVE");
Wallet w = wallet();
String gateway = trimGateway(cfg.arweaveGateway());
ArweaveMerkle.Prepared merkle = ArweaveMerkle.prepare(archiveFile);
String anchor = getRequiredText(gateway, "/tx_anchor");
String reward = getRequiredText(gateway, "/price/" + dataSize);
if (!reward.matches("^\\d+$")) throw new IOException("Arweave вернул некорректную цену: " + reward);
verifyBalance(gateway, w.address(), reward);
List<List<byte[]>> tagList = new ArrayList<>();
List<Map<String,String>> jsonTags = new ArrayList<>();
addTag(jsonTags, tagList, "Content-Type", "application/octet-stream");
addTag(jsonTags, tagList, "App-Name", "SHiNE");
addTag(jsonTags, tagList, "SHiNE-Type", "archive");
addTag(jsonTags, tagList, "SHiNE-Archive-Version", "1.0");
addTag(jsonTags, tagList, "SHiNE-Publisher", publisherLogin);
addTag(jsonTags, tagList, "SHiNE-Big-Block", Long.toUnsignedString(bigBlockNumber));
addTag(jsonTags, tagList, "SHiNE-Archive-SHA256", hex(archiveHash));
byte[] signaturePayload = deepHash(List.of(
utf8("2"),
b64Decode(w.owner()),
new byte[0],
utf8("0"),
utf8(reward),
b64Decode(anchor),
tagList,
utf8(Long.toString(dataSize)),
merkle.dataRoot()
));
byte[] rawSignature = signPayload(w.privateKey(), signaturePayload);
byte[] txIdBytes = sha256(rawSignature);
String txId = b64(txIdBytes);
Map<String,Object> tx = new LinkedHashMap<>();
tx.put("format", 2);
tx.put("id", txId);
tx.put("last_tx", anchor);
tx.put("owner", w.owner());
tx.put("tags", jsonTags);
tx.put("target", "");
tx.put("quantity", "0");
tx.put("data_root", b64(merkle.dataRoot()));
tx.put("data_size", Long.toString(dataSize));
tx.put("data", "");
tx.put("reward", reward);
tx.put("signature", b64(rawSignature));
postJsonExpect2xx(gateway + "/tx", tx, Duration.ofSeconds(60));
log.info("Arweave tx header принят: tx={}, chunks={}, bytes={}", txId, merkle.chunks().size(), dataSize);
for (int i=0; i<merkle.chunks().size(); i++) {
ArweaveMerkle.Chunk chunkMeta = merkle.chunks().get(i);
ArweaveMerkle.Proof proof = merkle.proofs().get(i);
byte[] chunk = ArweaveMerkle.readChunk(archiveFile, chunkMeta);
Map<String,Object> body = new LinkedHashMap<>();
body.put("data_root", b64(merkle.dataRoot()));
body.put("data_size", Long.toString(dataSize));
body.put("data_path", b64(proof.proof()));
body.put("offset", Long.toString(proof.offset()));
body.put("chunk", b64(chunk));
postChunkWithRetry(gateway, body, i, merkle.chunks().size());
}
return new UploadResult(txId, txIdBytes, gateway);
}
/** Ждёт включения tx в блок и заданного количества подтверждений. */
public int waitForConfirmations(String txId) throws Exception {
if (cfg.arweaveMinConfirmations() <= 0) return 0;
String gateway = trimGateway(cfg.arweaveGateway());
long deadline = System.currentTimeMillis() + cfg.arweaveConfirmTimeoutMinutes() * 60_000L;
int last = 0;
while (System.currentTimeMillis() < deadline) {
HttpRequest req = HttpRequest.newBuilder(URI.create(gateway + "/tx/" + txId + "/status"))
.timeout(Duration.ofSeconds(20)).GET().build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() == 200) {
JsonNode json = MAPPER.readTree(resp.body());
last = json.path("number_of_confirmations").asInt(0);
if (last >= cfg.arweaveMinConfirmations()) return last;
} else if (resp.statusCode() != 202 && resp.statusCode() != 404) {
log.warn("Arweave status tx={} HTTP {}: {}", txId, resp.statusCode(), safe(resp.body()));
}
Thread.sleep(cfg.arweaveConfirmPollSeconds() * 1000L);
}
throw new IOException("Истёк timeout ожидания Arweave confirmations для " + txId + ", last=" + last);
}
private void postChunkWithRetry(String gateway, Map<String,Object> body, int index, int total) throws Exception {
Exception last = null;
for (int attempt=1; attempt<=12; attempt++) {
try {
postJsonExpect2xx(gateway + "/chunk", body, Duration.ofSeconds(60));
if ((index + 1) % 25 == 0 || index + 1 == total) {
log.info("Arweave chunks: {}/{}", index + 1, total);
}
return;
} catch (Exception e) {
last = e;
if (attempt == 12) break;
log.warn("Arweave chunk {}/{} attempt {}/12: {}", index + 1, total, attempt, e.getMessage());
Thread.sleep(5_000L);
}
}
throw new IOException("Не удалось загрузить Arweave chunk " + (index + 1) + "/" + total, last);
}
private void postJsonExpect2xx(String url, Object body, Duration timeout) throws Exception {
String json = MAPPER.writeValueAsString(body);
HttpRequest req = HttpRequest.newBuilder(URI.create(url))
.timeout(timeout)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(json, StandardCharsets.UTF_8))
.build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() < 200 || resp.statusCode() >= 300) {
throw new IOException("HTTP " + resp.statusCode() + " " + safe(resp.body()));
}
}
private void verifyBalance(String gateway, String address, String reward) throws Exception {
if (address == null || address.isBlank()) return;
String balance = getRequiredText(gateway, "/wallet/" + address + "/balance");
if (balance.matches("^\\d+$") && new BigInteger(balance).compareTo(new BigInteger(reward)) < 0) {
throw new IllegalStateException("Недостаточно AR: balance=" + balance + ", reward=" + reward);
}
}
private String getRequiredText(String gateway, String path) throws Exception {
HttpRequest req = HttpRequest.newBuilder(URI.create(gateway + path)).timeout(Duration.ofSeconds(20)).GET().build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() < 200 || resp.statusCode() >= 300) {
throw new IOException("Arweave HTTP " + resp.statusCode() + " для " + path + ": " + safe(resp.body()));
}
String s = resp.body() == null ? "" : resp.body().trim();
if (s.isBlank()) throw new IOException("Пустой Arweave ответ для " + path);
return s;
}
private Wallet wallet() throws Exception {
Wallet local = wallet;
if (local != null) return local;
synchronized (this) {
if (wallet != null) return wallet;
JsonNode jwk = MAPPER.readTree(Files.readString(cfg.arweaveWalletJwkPath(), StandardCharsets.UTF_8));
String owner = required(jwk, "n");
PrivateKey pk = buildPrivateKey(jwk);
String address = b64(sha256(b64Decode(owner)));
wallet = new Wallet(owner, address, pk);
return wallet;
}
}
private static PrivateKey buildPrivateKey(JsonNode jwk) throws GeneralSecurityException {
RSAPrivateCrtKeySpec spec = new RSAPrivateCrtKeySpec(
big(required(jwk,"n")), big(required(jwk,"e")), big(required(jwk,"d")),
big(required(jwk,"p")), big(required(jwk,"q")), big(required(jwk,"dp")),
big(required(jwk,"dq")), big(required(jwk,"qi")));
return KeyFactory.getInstance("RSA").generatePrivate(spec);
}
private static byte[] signPayload(PrivateKey key, byte[] payload) throws GeneralSecurityException {
Signature signature = Signature.getInstance("RSASSA-PSS");
signature.setParameter(new PSSParameterSpec("SHA-256", "MGF1", MGF1ParameterSpec.SHA256, 32, 1));
signature.initSign(key);
signature.update(payload);
return signature.sign();
}
private static byte[] deepHash(Object data) throws GeneralSecurityException {
if (data instanceof List<?> list) {
byte[] tag = concat(utf8("list"), utf8(Integer.toString(list.size())));
return deepHashChunks(list, sha384(tag));
}
if (!(data instanceof byte[] bytes)) throw new IllegalArgumentException("deepHash: ожидается byte[]/List");
byte[] tag = concat(utf8("blob"), utf8(Integer.toString(bytes.length)));
return sha384(concat(sha384(tag), sha384(bytes)));
}
private static byte[] deepHashChunks(List<?> list, byte[] acc) throws GeneralSecurityException {
byte[] cur = acc;
for (Object item : list) cur = sha384(concat(cur, deepHash(item)));
return cur;
}
private static void addTag(List<Map<String,String>> json, List<List<byte[]>> raw, String name, String value) {
byte[] n=utf8(name), v=utf8(value);
Map<String,String> item = new LinkedHashMap<>();
item.put("name", b64(n)); item.put("value", b64(v));
json.add(item); raw.add(List.of(n,v));
}
private static String required(JsonNode node, String field) {
String s = node == null ? "" : node.path(field).asText("").trim();
if (s.isBlank()) throw new IllegalStateException("В Arweave JWK отсутствует " + field);
return s;
}
private static BigInteger big(String s) { return new BigInteger(1, b64Decode(s)); }
private static String trimGateway(String s) { return (s == null ? "https://arweave.net" : s.trim()).replaceAll("/+$", ""); }
private static byte[] utf8(String s) { return String.valueOf(s == null ? "" : s).getBytes(StandardCharsets.UTF_8); }
private static byte[] b64Decode(String s) { return B64D.decode(s); }
private static String b64(byte[] b) { return B64.encodeToString(b); }
private static byte[] sha256(byte[] b) throws GeneralSecurityException { return MessageDigest.getInstance("SHA-256").digest(b); }
private static byte[] sha384(byte[] b) throws GeneralSecurityException { return MessageDigest.getInstance("SHA-384").digest(b); }
private static String hex(byte[] b) { StringBuilder s=new StringBuilder(); for(byte x:b)s.append(String.format("%02x",x)); return s.toString(); }
private static byte[] concat(byte[]... arrays) { int n=0; for(byte[] a:arrays)n=Math.addExact(n,a.length); byte[] o=new byte[n]; int p=0; for(byte[]a:arrays){System.arraycopy(a,0,o,p,a.length);p+=a.length;} return o; }
private static String safe(String s) { s=String.valueOf(s==null?"":s).replace('\n',' ').replace('\r',' ').trim(); return s.length()<=250?s:s.substring(0,250)+"..."; }
private record Wallet(String owner, String address, PrivateKey privateKey) {}
}
@@ -0,0 +1,157 @@
package server.archive;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
/**
* Arweave v2 data Merkle tree/proofs. Алгоритм повторяет arweave-js merkle.ts.
*/
final class ArweaveMerkle {
static final int MAX_CHUNK_SIZE = 256 * 1024;
static final int MIN_CHUNK_SIZE = 32 * 1024;
record Chunk(byte[] dataHash, long minByteRange, long maxByteRange) {
long size() { return maxByteRange - minByteRange; }
}
record Proof(long offset, byte[] proof) {}
record Prepared(byte[] dataRoot, List<Chunk> chunks, List<Proof> proofs) {}
private sealed interface Node permits Leaf, Branch {
byte[] id();
long maxByteRange();
}
private record Leaf(byte[] id, byte[] dataHash, long minByteRange, long maxByteRange) implements Node {}
private record Branch(byte[] id, long byteRange, long maxByteRange, Node left, Node right) implements Node {}
private ArweaveMerkle() {}
static Prepared prepare(Path file) throws IOException {
long fileSize = Files.size(file);
List<Chunk> chunks = chunkFile(file, fileSize);
List<Node> leaves = new ArrayList<>(chunks.size());
for (Chunk c : chunks) {
byte[] id = sha256(concat(sha256(c.dataHash()), sha256(note(c.maxByteRange()))));
leaves.add(new Leaf(id, c.dataHash(), c.minByteRange(), c.maxByteRange()));
}
if (leaves.isEmpty()) throw new IOException("Arweave Merkle: нет leaves");
Node root = buildLayers(leaves);
List<Proof> proofs = new ArrayList<>();
resolveProofs(root, new byte[0], proofs);
// arweave-js строит root вместе с нулевым последним chunk при точном кратном 256KiB,
// но сам нулевой chunk/proof не загружает.
if (!chunks.isEmpty() && chunks.get(chunks.size() - 1).size() == 0) {
chunks = new ArrayList<>(chunks);
chunks.remove(chunks.size() - 1);
proofs.remove(proofs.size() - 1);
}
if (chunks.size() != proofs.size()) throw new IOException("Arweave Merkle: chunks/proofs mismatch");
return new Prepared(root.id(), List.copyOf(chunks), List.copyOf(proofs));
}
private static List<Chunk> chunkFile(Path file, long fileSize) throws IOException {
List<Chunk> out = new ArrayList<>();
long remaining = fileSize;
long cursor = 0;
try (InputStream in = new BufferedInputStream(Files.newInputStream(file))) {
while (remaining >= MAX_CHUNK_SIZE) {
long chunkSize = MAX_CHUNK_SIZE;
long next = remaining - MAX_CHUNK_SIZE;
if (next > 0 && next < MIN_CHUNK_SIZE) {
chunkSize = (remaining + 1) / 2;
}
byte[] data = readExact(in, (int)chunkSize);
out.add(new Chunk(sha256(data), cursor, cursor + chunkSize));
cursor += chunkSize;
remaining -= chunkSize;
}
byte[] rest = readExact(in, (int)remaining);
out.add(new Chunk(sha256(rest), cursor, cursor + remaining));
}
return out;
}
private static Node buildLayers(List<Node> nodes) {
if (nodes.size() < 2) return nodes.get(0);
List<Node> next = new ArrayList<>((nodes.size() + 1) / 2);
for (int i = 0; i < nodes.size(); i += 2) {
Node left = nodes.get(i);
Node right = i + 1 < nodes.size() ? nodes.get(i + 1) : null;
if (right == null) {
next.add(left);
continue;
}
long byteRange = left.maxByteRange();
byte[] id = sha256(concat(
sha256(left.id()),
sha256(right.id()),
sha256(note(byteRange))
));
next.add(new Branch(id, byteRange, right.maxByteRange(), left, right));
}
return buildLayers(next);
}
private static void resolveProofs(Node node, byte[] prefix, List<Proof> out) {
if (node instanceof Leaf leaf) {
out.add(new Proof(leaf.maxByteRange() - 1,
concat(prefix, leaf.dataHash(), note(leaf.maxByteRange()))));
return;
}
Branch b = (Branch) node;
byte[] partial = concat(prefix, b.left().id(), b.right().id(), note(b.byteRange()));
resolveProofs(b.left(), partial, out);
resolveProofs(b.right(), partial, out);
}
static byte[] readChunk(Path file, Chunk c) throws IOException {
long len = c.size();
if (len < 0 || len > Integer.MAX_VALUE) throw new IOException("Некорректный Arweave chunk size: " + len);
try (RandomAccessFile raf = new RandomAccessFile(file.toFile(), "r")) {
raf.seek(c.minByteRange());
byte[] data = new byte[(int)len];
raf.readFully(data);
return data;
}
}
private static byte[] readExact(InputStream in, int len) throws IOException {
byte[] out = new byte[len];
int pos = 0;
while (pos < len) {
int n = in.read(out, pos, len - pos);
if (n < 0) throw new EOFException("Неожиданный EOF");
pos += n;
}
return out;
}
private static byte[] note(long value) {
if (value < 0) throw new IllegalArgumentException("negative note");
byte[] out = new byte[32];
long v = value;
for (int i = 31; i >= 0 && v != 0; i--) {
out[i] = (byte)(v & 0xff);
v >>>= 8;
}
return out;
}
private static byte[] sha256(byte[] data) {
try { return MessageDigest.getInstance("SHA-256").digest(data); }
catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
}
private static byte[] concat(byte[]... arrays) {
int len = 0;
for (byte[] a : arrays) len = Math.addExact(len, a.length);
byte[] out = new byte[len];
int p=0;
for (byte[] a : arrays) { System.arraycopy(a,0,out,p,a.length); p+=a.length; }
return out;
}
}
@@ -0,0 +1,310 @@
package server.archive;
import shine.db.archive.ArchiveBigBlockRef;
import shine.db.archive.ArchivePublishJobChain;
import shine.db.entities.BlockEntry;
import utils.crypto.Ed25519Util;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
/**
* Writer бинарного SHINE-ARCHIVE v1.0.
*
* Один blockchain_name встречается максимум один раз и содержит все новые raw records этой цепочки.
*/
public final class ShineArchiveWriter {
public static final byte VERSION_MAJOR = 1;
public static final byte VERSION_MINOR = 0;
public static final byte REFERENCES_FULL = 0;
public static final long NO_REFERENCE_U32 = 0xffff_ffffL;
public static final int REFERENCE_ENTRY_SIZE = 68;
public static final byte[] MAGIC = "SHINE-ARCHIVE".getBytes(StandardCharsets.US_ASCII);
public static final byte[] SIGNATURE_DOMAIN = "SHINE-ARCHIVE-V1".getBytes(StandardCharsets.US_ASCII);
public record ChunkPosition(long offset, long size) {}
public record BuildResult(Path file, byte[] archiveHash, Map<String, ChunkPosition> chunks, long fileSize) {}
public BuildResult build(
Path finalPath,
String creatorLogin,
long bigBlockNumber,
long createdAtMs,
List<ArchiveBigBlockRef> previousBigBlocks,
List<ArchivePublishJobChain> chains,
Map<String, List<BlockEntry>> recordsByBlockchain,
byte[] signingPrivateSeed32,
long maxFileBytes
) throws IOException {
Objects.requireNonNull(finalPath, "finalPath");
Objects.requireNonNull(previousBigBlocks, "previousBigBlocks");
Objects.requireNonNull(chains, "chains");
Objects.requireNonNull(recordsByBlockchain, "recordsByBlockchain");
requireU32(bigBlockNumber, "big_block_number");
if (signingPrivateSeed32 == null || signingPrivateSeed32.length != 32) {
throw new IllegalArgumentException("Archive signing key должен быть 32-byte Ed25519 seed");
}
byte[] loginBytes = utf8U8(creatorLogin, "creator_login");
if (previousBigBlocks.size() > 0xffff_ffffL || chains.size() > 0xffff_ffffL) {
throw new IllegalArgumentException("Слишком много references/chunks для v1");
}
List<ArchivePublishJobChain> sortedChains = new ArrayList<>(chains);
sortedChains.sort(Comparator.comparing(ArchivePublishJobChain::blockchainName));
long totalRecords = 0;
for (ArchivePublishJobChain ch : sortedChains) {
List<BlockEntry> rs = recordsByBlockchain.get(ch.blockchainName());
if (rs == null || rs.isEmpty()) {
throw new IllegalArgumentException("Нет frozen records для " + ch.blockchainName());
}
totalRecords += rs.size();
requireU32(totalRecords, "total_user_records_count");
}
Map<Long, Integer> referenceIndexByBlock = new HashMap<>();
for (int i = 0; i < previousBigBlocks.size(); i++) {
ArchiveBigBlockRef ref = previousBigBlocks.get(i);
requireU32(ref.bigBlockNumber(), "PreviousBigBlockReference.big_block_number");
require32(ref.archiveHash(), "PreviousBigBlockReference.big_block_hash");
require32(ref.arweaveTxId(), "PreviousBigBlockReference.arweave_tx_id");
referenceIndexByBlock.put(ref.bigBlockNumber(), i);
}
Files.createDirectories(finalPath.toAbsolutePath().getParent());
Path tmp = finalPath.resolveSibling(finalPath.getFileName() + ".tmp");
Files.deleteIfExists(tmp);
Map<String, ChunkPosition> positions = new LinkedHashMap<>();
MessageDigest digest = sha256();
try (OutputStream fileOut = new BufferedOutputStream(Files.newOutputStream(tmp, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE));
CountingOutputStream counting = new CountingOutputStream(fileOut);
DigestOutputStream digestOut = new DigestOutputStream(counting, digest);
DataOutputStream out = new DataOutputStream(digestOut)) {
// ---------- header ----------
out.write(MAGIC);
out.writeByte(VERSION_MAJOR);
out.writeByte(VERSION_MINOR);
long headerSize = fixedHeaderSize() + loginBytes.length;
writeU32(out, headerSize);
writeU32(out, bigBlockNumber);
out.writeLong(createdAtMs);
out.writeByte(loginBytes.length);
out.writeByte(REFERENCES_FULL);
writeU32(out, previousBigBlocks.size());
out.writeShort(REFERENCE_ENTRY_SIZE);
long parentIndex = previousBigBlocks.isEmpty() ? NO_REFERENCE_U32 : previousBigBlocks.size() - 1L;
writeU32(out, parentIndex);
writeU32(out, sortedChains.size());
writeU32(out, totalRecords);
out.write(loginBytes);
// ---------- полный список предыдущих больших блоков ----------
for (ArchiveBigBlockRef ref : previousBigBlocks) {
writeU32(out, ref.bigBlockNumber());
out.write(ref.archiveHash());
out.write(ref.arweaveTxId());
}
// ---------- по одному chunk на blockchain_name ----------
for (ArchivePublishJobChain ch : sortedChains) {
List<BlockEntry> records = new ArrayList<>(recordsByBlockchain.get(ch.blockchainName()));
records.sort(Comparator.comparingInt(BlockEntry::getBlockNumber));
validateFrozenRange(ch, records);
byte[] name = utf8U8(ch.blockchainName(), "blockchain_name");
long chunkSize = calculateChunkSize(name.length, records);
requireU32(chunkSize, "chunk_size");
long chunkOffset = counting.getCount();
requireU32(chunkOffset, "chunk_offset");
if (chunkOffset + chunkSize > maxFileBytes) {
throw new ArchiveTooLargeException("Большой архив превысит archive.maxFileBytes: " + (chunkOffset + chunkSize));
}
writeU32(out, chunkSize);
out.writeByte(name.length);
out.write(name);
writeU32(out, records.size());
for (BlockEntry record : records) {
byte[] raw = record.getBlockBytes();
if (raw == null) throw new IllegalArgumentException("block_bytes=NULL: " + ch.blockchainName() + "#" + record.getBlockNumber());
writeU32(out, raw.length);
out.write(raw);
}
if (ch.previousArchiveBigBlockNumber() == null) {
writeU32(out, NO_REFERENCE_U32);
writeU32(out, 0);
writeU32(out, 0);
} else {
Integer refIndex = referenceIndexByBlock.get(ch.previousArchiveBigBlockNumber());
if (refIndex == null) {
throw new IllegalStateException("Предыдущий big block blockchain отсутствует в FULL reference table: "
+ ch.blockchainName() + " -> " + ch.previousArchiveBigBlockNumber());
}
ArchiveBigBlockRef previousRef = previousBigBlocks.get(refIndex);
require32(ch.previousArchiveBigBlockHash(), "previous_archive_big_block_hash " + ch.blockchainName());
if (!Arrays.equals(previousRef.archiveHash(), ch.previousArchiveBigBlockHash())) {
throw new IllegalStateException("Hash предыдущего big block не совпадает с FULL reference table: "
+ ch.blockchainName() + " -> " + ch.previousArchiveBigBlockNumber());
}
if (ch.previousChunkOffset() == null || ch.previousChunkSize() == null) {
throw new IllegalStateException("Нет предыдущих chunk offset/size: " + ch.blockchainName());
}
writeU32(out, refIndex);
writeU32(out, ch.previousChunkOffset());
writeU32(out, ch.previousChunkSize());
}
positions.put(ch.blockchainName(), new ChunkPosition(chunkOffset, chunkSize));
}
// ---------- footer, login входит в hash ----------
out.writeByte(loginBytes.length);
out.write(loginBytes);
out.flush();
byte[] archiveHash = digest.digest();
byte[] signedPayload = new byte[SIGNATURE_DOMAIN.length + archiveHash.length];
System.arraycopy(SIGNATURE_DOMAIN, 0, signedPayload, 0, SIGNATURE_DOMAIN.length);
System.arraycopy(archiveHash, 0, signedPayload, SIGNATURE_DOMAIN.length, archiveHash.length);
byte[] signature = Ed25519Util.sign(signedPayload, signingPrivateSeed32);
// hash/signature не должны попасть в hash самого блока.
digestOut.on(false);
out.write(archiveHash);
out.write(signature);
out.flush();
long finalSize = counting.getCount();
if (finalSize >= 0x1_0000_0000L || finalSize > maxFileBytes) {
throw new ArchiveTooLargeException("Размер SHINE-ARCHIVE v1 превышает лимит: " + finalSize);
}
} catch (Exception e) {
Files.deleteIfExists(tmp);
if (e instanceof IOException ioe) throw ioe;
throw new IOException("Не удалось собрать SHINE-ARCHIVE", e);
}
// До Arweave upload существует уже готовый локальный файл с окончательным именем.
try {
Files.move(tmp, finalPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException e) {
Files.move(tmp, finalPath, StandardCopyOption.REPLACE_EXISTING);
}
byte[] hash = readStoredArchiveHash(finalPath);
long size = Files.size(finalPath);
return new BuildResult(finalPath, hash, Collections.unmodifiableMap(positions), size);
}
/** Проверяет локальный файл перед оплатой Arweave. */
public static byte[] verifyLocalFile(Path file) throws IOException {
long size = Files.size(file);
if (size < MAGIC.length + 2 + 96) throw new IOException("Слишком короткий SHINE-ARCHIVE: " + file);
byte[] stored = readStoredArchiveHash(file);
MessageDigest md = sha256();
long hashInputSize = size - 96; // [archive_hash 32][signature 64]
try (InputStream in = Files.newInputStream(file)) {
byte[] buf = new byte[1024 * 1024];
long left = hashInputSize;
while (left > 0) {
int n = in.read(buf, 0, (int)Math.min(buf.length, left));
if (n < 0) throw new EOFException("Неожиданный EOF при проверке архива");
md.update(buf, 0, n);
left -= n;
}
}
byte[] actual = md.digest();
if (!Arrays.equals(stored, actual)) throw new IOException("SHA-256 локального SHINE-ARCHIVE не совпадает с footer");
return actual;
}
public static byte[] readStoredArchiveHash(Path file) throws IOException {
long size = Files.size(file);
if (size < 96) throw new EOFException("Файл слишком короткий");
try (RandomAccessFile raf = new RandomAccessFile(file.toFile(), "r")) {
raf.seek(size - 96);
byte[] hash = new byte[32];
raf.readFully(hash);
return hash;
}
}
private static long fixedHeaderSize() {
// magic + major/minor + headerSize + blockNo + time + creatorLen + mode + refCount + refSize + parentIndex + chunkCount + recordCount
return 13L + 1 + 1 + 4 + 4 + 8 + 1 + 1 + 4 + 2 + 4 + 4 + 4;
}
private static long calculateChunkSize(int nameLen, List<BlockEntry> records) {
long size = 4L + 1L + nameLen + 4L + 12L;
for (BlockEntry e : records) {
byte[] raw = e.getBlockBytes();
if (raw == null) throw new IllegalArgumentException("raw record is null");
size += 4L + raw.length;
}
return size;
}
private static void validateFrozenRange(ArchivePublishJobChain ch, List<BlockEntry> records) {
long expected = ch.toSourceBlockNumber() - ch.fromSourceBlockNumber() + 1;
if (expected <= 0 || expected != records.size()) {
throw new IllegalArgumentException("Frozen range неполон для " + ch.blockchainName()
+ ": ожидалось=" + expected + ", получено=" + records.size());
}
long n = ch.fromSourceBlockNumber();
for (BlockEntry e : records) {
if (Integer.toUnsignedLong(e.getBlockNumber()) != n) {
throw new IllegalArgumentException("Нарушена непрерывность " + ch.blockchainName()
+ ": ожидался block " + n + ", получен " + Integer.toUnsignedLong(e.getBlockNumber()));
}
n++;
}
}
private static byte[] utf8U8(String value, String name) {
if (value == null) throw new IllegalArgumentException(name + " == null");
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
if (bytes.length == 0 || bytes.length > 255) throw new IllegalArgumentException(name + " должен занимать 1..255 UTF-8 байт");
return bytes;
}
private static void require32(byte[] v, String name) {
if (v == null || v.length != 32) throw new IllegalArgumentException(name + " должен быть 32 bytes");
}
private static void requireU32(long v, String name) {
if (v < 0 || v > 0xffff_ffffL) throw new IllegalArgumentException(name + " вне u32: " + v);
}
private static void writeU32(DataOutputStream out, long v) throws IOException {
requireU32(v, "u32");
out.writeInt((int)v); // DataOutputStream = big-endian; битовый шаблон корректен и для unsigned > 2^31.
}
private static MessageDigest sha256() {
try { return MessageDigest.getInstance("SHA-256"); }
catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); }
}
/** OutputStream со счётчиком без ограничения int. */
private static final class CountingOutputStream extends FilterOutputStream {
private long count;
CountingOutputStream(OutputStream out) { super(out); }
long getCount() { return count; }
@Override public void write(int b) throws IOException { out.write(b); count++; }
@Override public void write(byte[] b, int off, int len) throws IOException { out.write(b, off, len); count += len; }
}
public static final class ArchiveTooLargeException extends IOException {
public ArchiveTooLargeException(String message) { super(message); }
}
}
@@ -0,0 +1,373 @@
package server.archive;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sync.codec.ShineUsersCodec;
import sync.codec.ShineUsersCodec.UserPdaSnapshot;
import sync.util.Base58Util;
import sync.util.SolanaPdaUtil;
import utils.crypto.Ed25519Util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.time.Duration;
import java.util.*;
/**
* Обновляет ArchiveHeadBlock (type=100) обычной инструкцией update_user_pda.
* Отдельной Solana-инструкции для архива нет.
*/
public final class SolanaArchiveHeadWriter {
private static final Logger log = LoggerFactory.getLogger(SolanaArchiveHeadWriter.class);
private static final ObjectMapper JSON = new ObjectMapper();
private static final String SYSTEM_PROGRAM = "11111111111111111111111111111111";
private static final String ED25519_PROGRAM = "Ed25519SigVerify111111111111111111111111111";
private static final String COMPUTE_BUDGET_PROGRAM = "ComputeBudget111111111111111111111111111111";
private static final String SYSVAR_INSTRUCTIONS = "Sysvar1nstructions1111111111111111111111111";
private static final String USER_SEED_PREFIX = "user_login=";
private static final String ECONOMY_SEED = "shine_users_economy_config";
private static final String INFLOW_SEED = "shine_payments_inflow_vault";
private static final byte[] LAST_BLOCK_STATE_PREFIX = "SHiNE_LAST_BLOCK".getBytes(StandardCharsets.UTF_8);
private final ArchivePublisherConfig cfg;
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(20)).build();
private final byte[] rootSeed32;
private final byte[] clientSeed32;
public SolanaArchiveHeadWriter(ArchivePublisherConfig cfg) throws IOException {
this.cfg = Objects.requireNonNull(cfg);
this.rootSeed32 = ArchiveKeyLoader.loadSeed32(cfg.solanaRootKeyPath());
this.clientSeed32 = ArchiveKeyLoader.loadSeed32(cfg.solanaClientKeyPath());
}
public record SubmitResult(String signature, String userPda) {}
/** Preflight: проверяет, что локальные root/client private keys соответствуют текущему User PDA. */
public void validateKeysAgainstPda() throws Exception {
Current current = loadCurrent();
byte[] expectedRoot = Base58Util.decode(current.snapshot.rootKey());
byte[] expectedClient = Base58Util.decode(current.snapshot.clientKey());
if (!Arrays.equals(expectedRoot, Ed25519Util.derivePublicKey(rootSeed32))) {
throw new IllegalStateException("archive.solana.rootKeyPath не соответствует root_key User PDA " + cfg.serverLogin());
}
if (!Arrays.equals(expectedClient, Ed25519Util.derivePublicKey(clientSeed32))) {
throw new IllegalStateException("archive.solana.clientKeyPath не соответствует client_key User PDA " + cfg.serverLogin());
}
}
public SubmitResult submitArchiveHead(byte[] arweaveTxId32, byte[] archiveHash32) throws Exception {
require32(arweaveTxId32, "Arweave TX ID");
require32(archiveHash32, "archive hash");
Current current = loadCurrent();
UserPdaSnapshot old = current.snapshot;
byte[] rootPub = Ed25519Util.derivePublicKey(rootSeed32);
byte[] clientPub = Ed25519Util.derivePublicKey(clientSeed32);
if (!Arrays.equals(rootPub, Base58Util.decode(old.rootKey()))) {
throw new IllegalStateException("Root key archive publisher-а не совпадает с User PDA");
}
if (!Arrays.equals(clientPub, Base58Util.decode(old.clientKey()))) {
throw new IllegalStateException("Client key archive publisher-а не совпадает с User PDA");
}
byte[] currentRaw = Base64.getDecoder().decode(old.rawDataBase64());
int currentRecordLen = u16le(currentRaw, 7);
byte[] currentUnsigned = Arrays.copyOf(currentRaw, currentRecordLen - 64);
byte[] prevHash = sha256(currentUnsigned);
long updatedAt = System.currentTimeMillis();
String txB64Url = Base64.getUrlEncoder().withoutPadding().encodeToString(arweaveTxId32);
String archiveHashHex = hex(archiveHash32);
String zeroSig = Base58Util.encode(new byte[64]);
UserPdaSnapshot nextUnsignedSnapshot = new UserPdaSnapshot(
old.pdaAddress(), old.login(), old.recordNumber() + 1, old.slot(), "",
old.recoveryKey(), old.rootKey(), old.clientKey(), old.blockchainName(), old.blockchainKey(),
old.paidLimitBytes(), old.usedBytes(), old.lastBlockNumber(), old.lastBlockHash(),
old.lastBlockSignature(), old.arweaveTxId(), txB64Url, archiveHashHex,
old.isServer(), old.addressFormatType(), old.addressFormatVersion(), old.serverAddress(),
old.syncServers(), old.accessServers(), old.sessionsMode(), old.sessions(), old.trustedCount(),
old.createdAtMs(), updatedAt, hex(prevHash), zeroSig, ""
);
byte[] nextFull = Base64.getDecoder().decode(ShineUsersCodec.serializeSnapshotToBase64(nextUnsignedSnapshot));
int nextRecordLen = u16le(nextFull, 7);
byte[] nextUnsigned = Arrays.copyOf(nextFull, nextRecordLen - 64);
byte[] nextUnsignedHash = sha256(nextUnsigned);
byte[] rootSignature = Ed25519Util.sign(nextUnsignedHash, rootSeed32);
byte[] updateData = buildUpdateInstruction(old, updatedAt, prevHash, arweaveTxId32, archiveHash32, rootSignature);
byte[] lastStateHash = sha256(buildLastBlockState(old));
byte[] lastBlockSignature = Base58Util.decode(old.lastBlockSignature());
byte[] blockchainPub = Base58Util.decode(old.blockchainKey());
byte[] rootEdData = buildEd25519Instruction(rootSignature, rootPub, nextUnsignedHash);
byte[] bchEdData = buildEd25519Instruction(lastBlockSignature, blockchainPub, lastStateHash);
String blockhash = getLatestBlockhash();
byte[] message = buildLegacyMessage(current.userPda, updateData, rootEdData, bchEdData, blockhash, clientPub);
byte[] clientSignature = Ed25519Util.sign(message, clientSeed32);
byte[] transaction = serializeTransaction(clientSignature, message);
String signature = sendTransaction(transaction);
log.info("Archive head отправлен в Solana: login={} pda={} signature={}", cfg.serverLogin(), current.userPda, signature);
return new SubmitResult(signature, current.userPda);
}
public void waitFinalized(String signature) throws Exception {
long deadline = System.currentTimeMillis() + cfg.solanaConfirmTimeoutMinutes() * 60_000L;
while (System.currentTimeMillis() < deadline) {
JsonNode result = rpc("getSignatureStatuses", List.of(List.of(signature), Map.of("searchTransactionHistory", true))).path("result");
JsonNode values = result.path("value");
JsonNode status = values.isArray() && values.size() > 0 ? values.get(0) : null;
if (status != null && !status.isNull()) {
if (status.hasNonNull("err")) {
throw new IOException("Solana archive-head tx завершилась ошибкой: " + status.path("err"));
}
String confirmation = status.path("confirmationStatus").asText("");
if ("finalized".equals(confirmation)) return;
}
Thread.sleep(cfg.solanaConfirmPollSeconds() * 1000L);
}
throw new IOException("Timeout ожидания Solana finalized: " + signature);
}
public boolean currentHeadMatches(byte[] txId32, byte[] hash32) throws Exception {
Current current = loadCurrent();
if (current.snapshot.archiveHeadTxId() == null || current.snapshot.archiveHeadTxId().isBlank()) return false;
byte[] currentTx = Base64.getUrlDecoder().decode(current.snapshot.archiveHeadTxId());
byte[] currentHash = unhex(current.snapshot.archiveHeadHash());
return Arrays.equals(currentTx, txId32) && Arrays.equals(currentHash, hash32);
}
private Current loadCurrent() throws Exception {
String normalized = cfg.serverLogin().trim().toLowerCase(Locale.ROOT);
String userPda = SolanaPdaUtil.findProgramAddress(
List.of(USER_SEED_PREFIX.getBytes(StandardCharsets.UTF_8), normalized.getBytes(StandardCharsets.UTF_8)),
cfg.solanaUsersProgramId());
JsonNode accountResult = rpc("getAccountInfo", List.of(userPda, Map.of("encoding", "base64", "commitment", cfg.solanaCommitment()))).path("result");
JsonNode value = accountResult.path("value");
if (value.isMissingNode() || value.isNull()) throw new IOException("User PDA не найден для " + cfg.serverLogin());
String owner = value.path("owner").asText("");
if (!cfg.solanaUsersProgramId().equals(owner)) throw new IOException("User PDA принадлежит другой программе: " + owner);
JsonNode data = value.path("data");
if (!data.isArray() || data.size() < 1) throw new IOException("Некорректный getAccountInfo.data");
String rawB64 = data.get(0).asText();
UserPdaSnapshot snapshot = ShineUsersCodec.parseUserPdaAccount(userPda, 0L, rawB64, "");
return new Current(userPda, snapshot);
}
private byte[] buildUpdateInstruction(UserPdaSnapshot s, long updatedAt, byte[] prevHash,
byte[] archiveTx, byte[] archiveHash, byte[] rootSig) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(4); // IX_UPDATE_USER_PDA
strU8(out, s.login());
out.write(Base58Util.decode(s.recoveryKey()));
out.write(Base58Util.decode(s.rootKey()));
u64le(out, s.createdAtMs());
u64le(out, updatedAt);
u32le(out, Integer.toUnsignedLong(s.recordNumber() + 1));
out.write(prevHash);
u64le(out, 0L); // additional_limit
out.write(Base58Util.decode(s.clientKey()));
out.write(Base58Util.decode(s.blockchainKey()));
strU8(out, s.blockchainName());
u64le(out, s.usedBytes());
u32le(out, Integer.toUnsignedLong(s.lastBlockNumber()));
out.write(unhex(s.lastBlockHash()));
out.write(Base58Util.decode(s.lastBlockSignature()));
strU8(out, s.arweaveTxId());
out.write(s.isServer() ? 1 : 0);
if (s.isServer()) {
out.write(s.addressFormatType() & 0xff);
out.write(s.addressFormatVersion() & 0xff);
strU8(out, s.serverAddress());
if (s.syncServers().size() > 255) throw new IOException("Слишком много sync_servers");
out.write(s.syncServers().size());
for (String v : s.syncServers()) strU8(out, v);
}
if (s.accessServers().size() > 255) throw new IOException("Слишком много access_servers");
out.write(s.accessServers().size());
for (String v : s.accessServers()) strU8(out, v);
out.write(s.sessionsMode() & 0xff);
if (s.sessions().size() > 255) throw new IOException("Слишком много sessions");
out.write(s.sessions().size());
for (ShineUsersCodec.UserSessionSnapshot session : s.sessions()) {
out.write(session.sessionType() & 0xff);
out.write(session.sessionVersion() & 0xff);
strU8(out, session.sessionName());
out.write(Base58Util.decode(session.sessionPubKey()));
}
out.write(s.trustedCount() & 0xff);
// Archive extension к обычному update_user_pda: marker=1 + tx32 + hash32.
out.write(1);
out.write(archiveTx);
out.write(archiveHash);
out.write(rootSig);
return out.toByteArray();
}
private static byte[] buildLastBlockState(UserPdaSnapshot s) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(LAST_BLOCK_STATE_PREFIX);
strU8(out, s.login());
strU8(out, s.blockchainName());
u32le(out, Integer.toUnsignedLong(s.lastBlockNumber()));
out.write(unhex(s.lastBlockHash()));
u64le(out, s.usedBytes());
return out.toByteArray();
}
private byte[] buildLegacyMessage(String userPda, byte[] updateData, byte[] rootEd, byte[] bchEd,
String recentBlockhash, byte[] clientPub) throws IOException {
String economy = SolanaPdaUtil.findProgramAddress(List.of(ECONOMY_SEED.getBytes(StandardCharsets.UTF_8)), cfg.solanaUsersProgramId());
String inflow = SolanaPdaUtil.findProgramAddress(List.of(INFLOW_SEED.getBytes(StandardCharsets.UTF_8)), cfg.solanaPaymentsProgramId());
List<Account> accounts = List.of(
new Account(Base58Util.encode(clientPub), true, true),
new Account(userPda, false, true),
new Account(inflow, false, true),
new Account(SYSTEM_PROGRAM, false, false),
new Account(SYSVAR_INSTRUCTIONS, false, false),
new Account(economy, false, false),
new Account(COMPUTE_BUDGET_PROGRAM, false, false),
new Account(ED25519_PROGRAM, false, false),
new Account(cfg.solanaUsersProgramId(), false, false)
);
Map<String,Integer> ix = new HashMap<>();
for (int i=0;i<accounts.size();i++) ix.put(accounts.get(i).pubkey, i);
List<CompiledInstruction> instructions = new ArrayList<>();
instructions.add(new CompiledInstruction(ix.get(COMPUTE_BUDGET_PROGRAM), new int[0], computeHeapData(262_144)));
instructions.add(new CompiledInstruction(ix.get(COMPUTE_BUDGET_PROGRAM), new int[0], computeLimitData(800_000)));
instructions.add(new CompiledInstruction(ix.get(ED25519_PROGRAM), new int[0], rootEd));
instructions.add(new CompiledInstruction(ix.get(ED25519_PROGRAM), new int[0], bchEd));
instructions.add(new CompiledInstruction(ix.get(cfg.solanaUsersProgramId()), new int[]{
ix.get(Base58Util.encode(clientPub)), ix.get(userPda), ix.get(SYSTEM_PROGRAM), ix.get(inflow),
ix.get(SYSVAR_INSTRUCTIONS), ix.get(economy)
}, updateData));
ByteArrayOutputStream out = new ByteArrayOutputStream();
out.write(1); // numRequiredSignatures
out.write(0); // numReadonlySignedAccounts
out.write(6); // readonly unsigned: system, sysvar, economy, compute, ed25519, users
shortVec(out, accounts.size());
for (Account a : accounts) {
byte[] pk = Base58Util.decode(a.pubkey);
if (pk.length != 32) throw new IOException("Solana pubkey !=32: " + a.pubkey);
out.write(pk);
}
byte[] bh = Base58Util.decode(recentBlockhash);
if (bh.length != 32) throw new IOException("recentBlockhash !=32");
out.write(bh);
shortVec(out, instructions.size());
for (CompiledInstruction ci : instructions) {
out.write(ci.programIdIndex);
shortVec(out, ci.accounts.length);
for (int a : ci.accounts) out.write(a);
shortVec(out, ci.data.length);
out.write(ci.data);
}
return out.toByteArray();
}
private String getLatestBlockhash() throws Exception {
JsonNode result = rpc("getLatestBlockhash", List.of(Map.of("commitment", cfg.solanaCommitment()))).path("result");
String blockhash = result.path("value").path("blockhash").asText("");
if (blockhash.isBlank()) throw new IOException("Solana getLatestBlockhash не вернул blockhash");
return blockhash;
}
private String sendTransaction(byte[] tx) throws Exception {
String txB64 = Base64.getEncoder().encodeToString(tx);
JsonNode result = rpc("sendTransaction", List.of(txB64, Map.of(
"encoding", "base64",
"skipPreflight", false,
"preflightCommitment", "confirmed",
"maxRetries", 5
))).path("result");
String sig = result.asText("");
if (sig.isBlank()) throw new IOException("Solana sendTransaction не вернул signature");
return sig;
}
private JsonNode rpc(String method, List<?> params) throws Exception {
Map<String,Object> body = new LinkedHashMap<>();
body.put("jsonrpc", "2.0"); body.put("id", 1); body.put("method", method); body.put("params", params);
HttpRequest req = HttpRequest.newBuilder(URI.create(cfg.solanaRpcUrl()))
.timeout(Duration.ofSeconds(60))
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(JSON.writeValueAsString(body), StandardCharsets.UTF_8))
.build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() < 200 || resp.statusCode() >= 300) throw new IOException("Solana RPC HTTP " + resp.statusCode() + ": " + safe(resp.body()));
JsonNode root = JSON.readTree(resp.body());
if (root.hasNonNull("error")) throw new IOException("Solana RPC " + method + " error: " + root.path("error"));
return root;
}
private static byte[] serializeTransaction(byte[] signature64, byte[] message) throws IOException {
if (signature64.length != 64) throw new IOException("Solana signature !=64");
ByteArrayOutputStream out = new ByteArrayOutputStream();
shortVec(out, 1);
out.write(signature64);
out.write(message);
return out.toByteArray();
}
private static byte[] buildEd25519Instruction(byte[] signature64, byte[] publicKey32, byte[] message) {
if (signature64.length != 64 || publicKey32.length != 32) throw new IllegalArgumentException("Некорректный Ed25519 ix key/signature");
int sigOff=16, pkOff=sigOff+64, msgOff=pkOff+32;
if (message.length > 0xffff) throw new IllegalArgumentException("Ed25519 message слишком длинный");
byte[] data = new byte[msgOff + message.length];
data[0]=1; data[1]=0;
putU16le(data,2,sigOff); putU16le(data,4,0xffff);
putU16le(data,6,pkOff); putU16le(data,8,0xffff);
putU16le(data,10,msgOff); putU16le(data,12,message.length); putU16le(data,14,0xffff);
System.arraycopy(signature64,0,data,sigOff,64);
System.arraycopy(publicKey32,0,data,pkOff,32);
System.arraycopy(message,0,data,msgOff,message.length);
return data;
}
private static byte[] computeHeapData(int bytes) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(1); u32le(out, Integer.toUnsignedLong(bytes)); return out.toByteArray();
}
private static byte[] computeLimitData(int units) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(2); u32le(out, Integer.toUnsignedLong(units)); return out.toByteArray();
}
private static void shortVec(ByteArrayOutputStream out, int value) {
int rem=value;
while (true) {
int elem=rem & 0x7f; rem >>>=7;
if (rem==0) { out.write(elem); return; }
out.write(elem | 0x80);
}
}
private static void strU8(ByteArrayOutputStream out, String value) throws IOException {
byte[] b=String.valueOf(value==null?"":value).getBytes(StandardCharsets.UTF_8);
if (b.length>255) throw new IOException("Строка >255 bytes"); out.write(b.length); out.write(b);
}
private static void u32le(ByteArrayOutputStream out, long v) { for(int i=0;i<4;i++) out.write((int)(v >>> (8*i)) & 0xff); }
private static void u64le(ByteArrayOutputStream out, long v) { for(int i=0;i<8;i++) out.write((int)(v >>> (8*i)) & 0xff); }
private static void putU16le(byte[] d,int o,int v){d[o]=(byte)v;d[o+1]=(byte)(v>>>8);}
private static int u16le(byte[] b,int o){return (b[o]&255)|((b[o+1]&255)<<8);}
private static byte[] sha256(byte[] bytes) throws Exception { return MessageDigest.getInstance("SHA-256").digest(bytes); }
private static String hex(byte[] b){StringBuilder s=new StringBuilder(b.length*2);for(byte x:b)s.append(String.format("%02x",x));return s.toString();}
private static byte[] unhex(String s){String x=String.valueOf(s==null?"":s).trim(); if(x.length()!=64) throw new IllegalArgumentException("hex32 expected");byte[]o=new byte[32];for(int i=0;i<32;i++)o[i]=(byte)Integer.parseInt(x.substring(i*2,i*2+2),16);return o;}
private static void require32(byte[] b,String n){if(b==null||b.length!=32)throw new IllegalArgumentException(n+" должен быть 32 bytes");}
private static String safe(String s){String x=String.valueOf(s==null?"":s).replace('\n',' ').replace('\r',' ').trim();return x.length()<=500?x:x.substring(0,500)+"...";}
private record Current(String userPda, UserPdaSnapshot snapshot) {}
private record Account(String pubkey, boolean signer, boolean writable) {}
private record CompiledInstruction(int programIdIndex, int[] accounts, byte[] data) {}
}
@@ -0,0 +1,34 @@
package server.archive;
import org.junit.jupiter.api.Test;
import java.time.Instant;
import java.time.ZoneId;
import java.util.Base64;
import static org.junit.jupiter.api.Assertions.assertEquals;
class ArchiveFileNamesTest {
private static final long CREATED_AT = Instant.parse("2026-09-11T00:00:00Z").toEpochMilli();
private static final ZoneId UTC = ZoneId.of("UTC");
@Test
void usesTemporaryAndFinalArweaveNames() {
byte[] tx = new byte[32];
for (int i = 0; i < tx.length; i++) tx[i] = (byte) i;
String txText = Base64.getUrlEncoder().withoutPadding().encodeToString(tx);
assertEquals("archive01.00001.11.09.26.tmp.SHiNE-archive",
ArchiveFileNames.pendingFileName("archive01", 1, CREATED_AT, UTC));
assertEquals("archive01.00001.11.09.26." + txText + ".SHiNE-archive",
ArchiveFileNames.uploadedFileName("archive01", 1, CREATED_AT, UTC, tx));
assertEquals("archive01.100000.11.09.26.tmp.SHiNE-archive",
ArchiveFileNames.pendingFileName("archive01", 100000, CREATED_AT, UTC));
}
@Test
void sanitizesLoginForFileSystemOnly() {
assertEquals("archive_user.00002.11.09.26.tmp.SHiNE-archive",
ArchiveFileNames.pendingFileName("archive/user", 2, CREATED_AT, UTC));
}
}
@@ -0,0 +1,83 @@
package server.archive;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import shine.db.archive.ArchiveBigBlockRef;
import shine.db.archive.ArchivePublishJobChain;
import shine.db.entities.BlockEntry;
import java.io.DataInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class ShineArchiveWriterTest {
@TempDir Path dir;
@Test
void writesOneChunkWithAllRecordsAndOneBacklink() throws Exception {
Path file = ArchiveFileNames.pendingFilePath(dir, "serverA", 2, 123456789L, ZoneId.of("UTC"));
ArchiveBigBlockRef previous = new ArchiveBigBlockRef(1, new byte[32], new byte[32]);
ArchivePublishJobChain chain = new ArchivePublishJobChain(
1, "alice-001", 10, 11, new byte[32], new byte[32],
1L, new byte[32], 100L, 55L, null, null);
ShineArchiveWriter.BuildResult result = new ShineArchiveWriter().build(
file, "serverA", 2, 123456789L,
List.of(previous), List.of(chain),
Map.of("alice-001", List.of(block(10, new byte[]{1,2,3}), block(11, new byte[]{4,5}))),
new byte[32], 4_000_000_000L);
assertArrayEquals(result.archiveHash(), ShineArchiveWriter.verifyLocalFile(file));
assertEquals("serverA.00002.02.01.70.tmp.SHiNE-archive", file.getFileName().toString());
try (InputStream raw = java.nio.file.Files.newInputStream(file); DataInputStream in = new DataInputStream(raw)) {
assertEquals("SHINE-ARCHIVE", new String(in.readNBytes(13), StandardCharsets.US_ASCII));
assertEquals(1, in.readUnsignedByte());
assertEquals(0, in.readUnsignedByte());
long headerSize = Integer.toUnsignedLong(in.readInt());
assertEquals(2, Integer.toUnsignedLong(in.readInt()));
assertEquals(123456789L, in.readLong());
int creatorLen = in.readUnsignedByte();
assertEquals(ShineArchiveWriter.REFERENCES_FULL, in.readUnsignedByte());
assertEquals(1, Integer.toUnsignedLong(in.readInt()));
assertEquals(68, in.readUnsignedShort());
assertEquals(0, Integer.toUnsignedLong(in.readInt()));
assertEquals(1, Integer.toUnsignedLong(in.readInt()));
assertEquals(2, Integer.toUnsignedLong(in.readInt()));
assertEquals("serverA", new String(in.readNBytes(creatorLen), StandardCharsets.UTF_8));
assertEquals(headerSize, 51L + creatorLen);
assertEquals(1, Integer.toUnsignedLong(in.readInt()));
in.skipNBytes(64); // previous hash + txid
long chunkSize = Integer.toUnsignedLong(in.readInt());
int nameLen = in.readUnsignedByte();
assertEquals("alice-001", new String(in.readNBytes(nameLen), StandardCharsets.UTF_8));
assertEquals(2, Integer.toUnsignedLong(in.readInt()));
assertEquals(3, Integer.toUnsignedLong(in.readInt()));
assertArrayEquals(new byte[]{1,2,3}, in.readNBytes(3));
assertEquals(2, Integer.toUnsignedLong(in.readInt()));
assertArrayEquals(new byte[]{4,5}, in.readNBytes(2));
assertEquals(0, Integer.toUnsignedLong(in.readInt())); // previous ref index
assertEquals(100, Integer.toUnsignedLong(in.readInt()));
assertEquals(55, Integer.toUnsignedLong(in.readInt()));
assertEquals(4L + 1 + nameLen + 4 + (4+3) + (4+2) + 12, chunkSize);
}
}
private static BlockEntry block(int number, byte[] bytes) {
BlockEntry e = new BlockEntry();
e.setBlockNumber(number);
e.setBlockBytes(bytes);
e.setBlockHash(new byte[32]);
e.setBlockSignature(new byte[64]);
return e;
}
}
@@ -66,19 +66,20 @@ public final class AppConfig {
/** Вернёт строку или пустую строку, если параметр не найден. */
public String getStringOrEmpty(String name) {
String value = properties.getProperty(name);
String value = getParam(name);
return value == null ? "" : value.trim();
}
/** Можно добавить методы для удобства */
/** Целочисленный параметр с тем же приоритетом: system property -> env -> properties. */
public int getInt(String name, int defaultValue) {
String v = properties.getProperty(name);
return v == null ? defaultValue : Integer.parseInt(v);
String v = getParam(name);
return v == null || v.isBlank() ? defaultValue : Integer.parseInt(v.trim());
}
/** Boolean-параметр с тем же приоритетом: system property -> env -> properties. */
public boolean getBoolean(String name, boolean defaultValue) {
String v = properties.getProperty(name);
return v == null ? defaultValue : Boolean.parseBoolean(v);
String v = getParam(name);
return v == null || v.isBlank() ? defaultValue : Boolean.parseBoolean(v.trim());
}
private static String toEnvName(String name) {
@@ -38,6 +38,7 @@ public final class DatabaseInitializer {
public static final int SCHEMA_VERSION_19 = 19;
public static final int SCHEMA_VERSION_20 = 20;
public static final int SCHEMA_VERSION_21 = 21;
public static final int SCHEMA_VERSION_22 = 22;
public static final String POSTGRES_SCHEMA_RESOURCE = "postgres/schema_v1.sql";
public static final String POSTGRES_MIGRATION_V2_RESOURCE = "postgres/migration_v2.sql";
public static final String POSTGRES_MIGRATION_V3_RESOURCE = "postgres/migration_v3.sql";
@@ -59,6 +60,7 @@ public final class DatabaseInitializer {
public static final String POSTGRES_MIGRATION_V19_RESOURCE = "postgres/migration_v19.sql";
public static final String POSTGRES_MIGRATION_V20_RESOURCE = "postgres/migration_v20.sql";
public static final String POSTGRES_MIGRATION_V21_RESOURCE = "postgres/migration_v21.sql";
public static final String POSTGRES_MIGRATION_V22_RESOURCE = "postgres/migration_v22.sql";
private DatabaseInitializer() {}
@@ -212,6 +214,10 @@ public final class DatabaseInitializer {
runSqlScript(conn, POSTGRES_MIGRATION_V21_RESOURCE);
currentVersion = SCHEMA_VERSION_21;
}
if (currentVersion < SCHEMA_VERSION_22) {
runSqlScript(conn, POSTGRES_MIGRATION_V22_RESOURCE);
currentVersion = SCHEMA_VERSION_22;
}
}
}
@@ -0,0 +1,8 @@
package shine.db.archive;
/** Ссылка на уже успешно опубликованный большой архивный блок. */
public record ArchiveBigBlockRef(
long bigBlockNumber,
byte[] archiveHash,
byte[] arweaveTxId
) {}
@@ -0,0 +1,15 @@
package shine.db.archive;
/**
* Последнее окончательно опубликованное архивное положение одной SHiNE blockchain.
*/
public record ArchiveChainCursor(
String blockchainName,
long lastArchivedSourceBlockNumber,
byte[] lastArchivedSourceBlockHash,
long lastArchiveBigBlockNumber,
byte[] lastArchiveBigBlockHash,
long lastChunkOffset,
long lastChunkSize,
long updatedAtMs
) {}
@@ -0,0 +1,16 @@
package shine.db.archive;
/** Crash-safe состояние публикации одного большого SHINE-ARCHIVE блока. */
public record ArchivePublishJob(
long id,
long bigBlockNumber,
String status,
long createdAtMs,
String localArchivePath,
byte[] archiveHash,
byte[] arweaveTxId,
Integer arweaveConfirmations,
String solanaSignature,
String errorText,
long updatedAtMs
) {}
@@ -0,0 +1,17 @@
package shine.db.archive;
/** Замороженный диапазон одной blockchain внутри конкретной архивной публикации. */
public record ArchivePublishJobChain(
long jobId,
String blockchainName,
long fromSourceBlockNumber,
long toSourceBlockNumber,
byte[] previousSourceBlockHash,
byte[] lastSourceBlockHash,
Long previousArchiveBigBlockNumber,
byte[] previousArchiveBigBlockHash,
Long previousChunkOffset,
Long previousChunkSize,
Long newChunkOffset,
Long newChunkSize
) {}
@@ -0,0 +1,472 @@
package shine.db.dao;
import shine.db.DbController;
import shine.db.archive.ArchiveBigBlockRef;
import shine.db.archive.ArchiveChainCursor;
import shine.db.archive.ArchivePublishJob;
import shine.db.archive.ArchivePublishJobChain;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
/**
* DAO архивного publisher-а. Все курсоры считаются подтверждёнными только после Solana finalized.
*/
public final class ArchivePublicationDAO {
private static volatile ArchivePublicationDAO instance;
private final DbController db = DbController.getInstance();
private ArchivePublicationDAO() {}
public static ArchivePublicationDAO getInstance() {
if (instance == null) {
synchronized (ArchivePublicationDAO.class) {
if (instance == null) instance = new ArchivePublicationDAO();
}
}
return instance;
}
public ArchiveChainCursor getCursor(String blockchainName) throws SQLException {
try (Connection c = db.getConnection()) {
return getCursor(c, blockchainName);
}
}
public ArchiveChainCursor getCursor(Connection c, String blockchainName) throws SQLException {
String sql = """
SELECT blockchain_name, last_archived_source_block_number, last_archived_source_block_hash,
last_archive_big_block_number, last_archive_big_block_hash,
last_chunk_offset, last_chunk_size, updated_at_ms
FROM archive_chain_cursor
WHERE blockchain_name = ?
""";
try (PreparedStatement ps = c.prepareStatement(sql)) {
ps.setString(1, blockchainName);
try (ResultSet rs = ps.executeQuery()) {
if (!rs.next()) return null;
return mapCursor(rs);
}
}
}
public long nextBigBlockNumber() throws SQLException {
String sql = """
SELECT COALESCE(MAX(big_block_number), 0) + 1
FROM archive_publish_job
WHERE status = 'CURSORS_COMMITTED'
""";
try (Connection c = db.getConnection();
PreparedStatement ps = c.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
rs.next();
return rs.getLong(1);
}
}
public List<ArchiveBigBlockRef> listFinalizedBigBlocks() throws SQLException {
String sql = """
SELECT big_block_number, archive_hash, arweave_tx_id
FROM archive_publish_job
WHERE status = 'CURSORS_COMMITTED'
AND archive_hash IS NOT NULL
AND arweave_tx_id IS NOT NULL
ORDER BY big_block_number ASC
""";
List<ArchiveBigBlockRef> out = new ArrayList<>();
try (Connection c = db.getConnection();
PreparedStatement ps = c.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
out.add(new ArchiveBigBlockRef(
rs.getLong("big_block_number"),
rs.getBytes("archive_hash"),
rs.getBytes("arweave_tx_id")
));
}
}
return out;
}
public ArchivePublishJob findUnfinishedJob() throws SQLException {
String sql = """
SELECT id, big_block_number, status, created_at_ms, local_archive_path,
archive_hash, arweave_tx_id, arweave_confirmations, solana_signature,
error_text, updated_at_ms
FROM archive_publish_job
WHERE status NOT IN ('CURSORS_COMMITTED', 'FAILED')
ORDER BY id ASC
LIMIT 1
""";
try (Connection c = db.getConnection();
PreparedStatement ps = c.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
return rs.next() ? mapJob(rs) : null;
}
}
public ArchivePublishJob getJob(long id) throws SQLException {
String sql = """
SELECT id, big_block_number, status, created_at_ms, local_archive_path,
archive_hash, arweave_tx_id, arweave_confirmations, solana_signature,
error_text, updated_at_ms
FROM archive_publish_job
WHERE id = ?
""";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, id);
try (ResultSet rs = ps.executeQuery()) {
return rs.next() ? mapJob(rs) : null;
}
}
}
/** Атомарно создаёт frozen archive job и все его blockchain ranges. */
public long createJobWithChains(long bigBlockNumber, long createdAtMs, List<ArchivePublishJobChain> chains) throws SQLException {
if (chains == null || chains.isEmpty()) throw new IllegalArgumentException("Archive job без chains не создаётся");
try (Connection c = db.getConnection()) {
boolean oldAuto = c.getAutoCommit();
c.setAutoCommit(false);
try {
long jobId;
try (PreparedStatement ps = c.prepareStatement("""
INSERT INTO archive_publish_job(big_block_number, status, created_at_ms, updated_at_ms)
VALUES(?, 'SNAPSHOT_CREATED', ?, ?)
RETURNING id
""")) {
ps.setLong(1, bigBlockNumber);
ps.setLong(2, createdAtMs);
ps.setLong(3, createdAtMs);
try (ResultSet rs = ps.executeQuery()) {
if (!rs.next()) throw new SQLException("Не удалось получить id archive_publish_job");
jobId = rs.getLong(1);
}
}
String sql = """
INSERT INTO archive_publish_job_chain(
job_id, blockchain_name, from_source_block_number, to_source_block_number,
previous_source_block_hash, last_source_block_hash,
previous_archive_big_block_number, previous_archive_big_block_hash,
previous_chunk_offset, previous_chunk_size, new_chunk_offset, new_chunk_size
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
""";
try (PreparedStatement ps = c.prepareStatement(sql)) {
for (ArchivePublishJobChain ch : chains) {
bindJobChain(ps, new ArchivePublishJobChain(
jobId, ch.blockchainName(), ch.fromSourceBlockNumber(), ch.toSourceBlockNumber(),
ch.previousSourceBlockHash(), ch.lastSourceBlockHash(),
ch.previousArchiveBigBlockNumber(), ch.previousArchiveBigBlockHash(),
ch.previousChunkOffset(), ch.previousChunkSize(), null, null));
ps.addBatch();
}
ps.executeBatch();
}
c.commit();
return jobId;
} catch (Exception e) {
c.rollback();
if (e instanceof SQLException se) throw se;
throw new SQLException("Не удалось создать frozen archive job", e);
} finally {
c.setAutoCommit(oldAuto);
}
}
}
public long createJob(long bigBlockNumber, long createdAtMs) throws SQLException {
String sql = """
INSERT INTO archive_publish_job(big_block_number, status, created_at_ms, updated_at_ms)
VALUES(?, 'SNAPSHOT_CREATED', ?, ?)
RETURNING id
""";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, bigBlockNumber);
ps.setLong(2, createdAtMs);
ps.setLong(3, createdAtMs);
try (ResultSet rs = ps.executeQuery()) {
if (!rs.next()) throw new SQLException("Не удалось получить id archive_publish_job");
return rs.getLong(1);
}
}
}
public void insertJobChain(ArchivePublishJobChain e) throws SQLException {
String sql = """
INSERT INTO archive_publish_job_chain(
job_id, blockchain_name, from_source_block_number, to_source_block_number,
previous_source_block_hash, last_source_block_hash,
previous_archive_big_block_number, previous_archive_big_block_hash,
previous_chunk_offset, previous_chunk_size,
new_chunk_offset, new_chunk_size
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
""";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
bindJobChain(ps, e);
ps.executeUpdate();
}
}
public List<ArchivePublishJobChain> listJobChains(long jobId) throws SQLException {
String sql = """
SELECT job_id, blockchain_name, from_source_block_number, to_source_block_number,
previous_source_block_hash, last_source_block_hash,
previous_archive_big_block_number, previous_archive_big_block_hash,
previous_chunk_offset, previous_chunk_size,
new_chunk_offset, new_chunk_size
FROM archive_publish_job_chain
WHERE job_id = ?
ORDER BY blockchain_name ASC
""";
List<ArchivePublishJobChain> out = new ArrayList<>();
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, jobId);
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) out.add(mapJobChain(rs));
}
}
return out;
}
public void updateNewChunkPosition(long jobId, String blockchainName, long offset, long size) throws SQLException {
String sql = """
UPDATE archive_publish_job_chain
SET new_chunk_offset=?, new_chunk_size=?
WHERE job_id=? AND blockchain_name=?
""";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, offset);
ps.setLong(2, size);
ps.setLong(3, jobId);
ps.setString(4, blockchainName);
if (ps.executeUpdate() != 1) throw new SQLException("Не найдена archive_publish_job_chain для " + blockchainName);
}
}
public void markFileBuilt(long jobId, String localPath, byte[] hash) throws SQLException {
updateJob(jobId, "FILE_BUILT", "local_archive_path=? , archive_hash=?", ps -> {
ps.setString(1, localPath);
ps.setBytes(2, hash);
});
}
public void markArweaveUploaded(long jobId, byte[] txId) throws SQLException {
updateJob(jobId, "ARWEAVE_UPLOADED", "arweave_tx_id=?", ps -> ps.setBytes(1, txId));
}
/** Обновляет только локальное имя/путь уже собранного archive-файла. */
public void updateLocalArchivePath(long jobId, String localPath) throws SQLException {
updateJob(jobId, null, "local_archive_path=?", ps -> ps.setString(1, localPath));
}
public void markArweaveConfirmed(long jobId, int confirmations) throws SQLException {
updateJob(jobId, "ARWEAVE_CONFIRMED", "arweave_confirmations=?", ps -> ps.setInt(1, confirmations));
}
public void markSolanaSubmitted(long jobId, String signature) throws SQLException {
updateJob(jobId, "SOLANA_SUBMITTED", "solana_signature=?", ps -> ps.setString(1, signature));
}
public void markSolanaFinalized(long jobId) throws SQLException {
updateJob(jobId, "SOLANA_FINALIZED", null, null);
}
public void markFailed(long jobId, String error) throws SQLException {
updateJob(jobId, "FAILED", "error_text=?", ps -> ps.setString(1, error == null ? "" : error));
}
/**
* После Solana finalized атомарно переводит курсоры всех blockchain на новый chunk и закрывает job.
*/
public void commitCursors(long jobId, byte[] bigBlockHash) throws SQLException {
try (Connection c = db.getConnection()) {
boolean oldAuto = c.getAutoCommit();
c.setAutoCommit(false);
try {
ArchivePublishJob job = getJob(c, jobId);
if (job == null) throw new SQLException("Archive job не найден: " + jobId);
List<ArchivePublishJobChain> chains = listJobChains(c, jobId);
long now = System.currentTimeMillis();
String upsert = """
INSERT INTO archive_chain_cursor(
blockchain_name, last_archived_source_block_number, last_archived_source_block_hash,
last_archive_big_block_number, last_archive_big_block_hash,
last_chunk_offset, last_chunk_size, updated_at_ms
) VALUES(?,?,?,?,?,?,?,?)
ON CONFLICT(blockchain_name) DO UPDATE SET
last_archived_source_block_number=EXCLUDED.last_archived_source_block_number,
last_archived_source_block_hash=EXCLUDED.last_archived_source_block_hash,
last_archive_big_block_number=EXCLUDED.last_archive_big_block_number,
last_archive_big_block_hash=EXCLUDED.last_archive_big_block_hash,
last_chunk_offset=EXCLUDED.last_chunk_offset,
last_chunk_size=EXCLUDED.last_chunk_size,
updated_at_ms=EXCLUDED.updated_at_ms
""";
try (PreparedStatement ps = c.prepareStatement(upsert)) {
for (ArchivePublishJobChain ch : chains) {
if (ch.newChunkOffset() == null || ch.newChunkSize() == null) {
throw new SQLException("Не сохранены координаты нового chunk: " + ch.blockchainName());
}
ps.setString(1, ch.blockchainName());
ps.setLong(2, ch.toSourceBlockNumber());
ps.setBytes(3, ch.lastSourceBlockHash());
ps.setLong(4, job.bigBlockNumber());
ps.setBytes(5, bigBlockHash);
ps.setLong(6, ch.newChunkOffset());
ps.setLong(7, ch.newChunkSize());
ps.setLong(8, now);
ps.addBatch();
}
ps.executeBatch();
}
try (PreparedStatement ps = c.prepareStatement(
"UPDATE archive_publish_job SET status='CURSORS_COMMITTED', updated_at_ms=? WHERE id=?")) {
ps.setLong(1, now);
ps.setLong(2, jobId);
ps.executeUpdate();
}
c.commit();
} catch (Exception e) {
c.rollback();
if (e instanceof SQLException se) throw se;
throw new SQLException("Не удалось commit archive cursors", e);
} finally {
c.setAutoCommit(oldAuto);
}
}
}
private ArchivePublishJob getJob(Connection c, long id) throws SQLException {
String sql = """
SELECT id, big_block_number, status, created_at_ms, local_archive_path,
archive_hash, arweave_tx_id, arweave_confirmations, solana_signature,
error_text, updated_at_ms
FROM archive_publish_job WHERE id=?
""";
try (PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, id);
try (ResultSet rs = ps.executeQuery()) { return rs.next() ? mapJob(rs) : null; }
}
}
private List<ArchivePublishJobChain> listJobChains(Connection c, long jobId) throws SQLException {
String sql = """
SELECT job_id, blockchain_name, from_source_block_number, to_source_block_number,
previous_source_block_hash, last_source_block_hash,
previous_archive_big_block_number, previous_archive_big_block_hash,
previous_chunk_offset, previous_chunk_size,
new_chunk_offset, new_chunk_size
FROM archive_publish_job_chain
WHERE job_id=? ORDER BY blockchain_name ASC
""";
List<ArchivePublishJobChain> out = new ArrayList<>();
try (PreparedStatement ps = c.prepareStatement(sql)) {
ps.setLong(1, jobId);
try (ResultSet rs = ps.executeQuery()) { while (rs.next()) out.add(mapJobChain(rs)); }
}
return out;
}
@FunctionalInterface
private interface Binder { void bind(PreparedStatement ps) throws SQLException; }
private void updateJob(long jobId, String status, String setFragment, Binder binder) throws SQLException {
if (setFragment != null && !setFragment.isBlank()) {
try (Connection c = db.getConnection()) {
updateJobWithExtra(c, jobId, status, setFragment, binder);
}
return;
}
String sql = status == null
? "UPDATE archive_publish_job SET updated_at_ms=? WHERE id=?"
: "UPDATE archive_publish_job SET status=?, updated_at_ms=? WHERE id=?";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
int i = 1;
if (status != null) ps.setString(i++, status);
ps.setLong(i++, System.currentTimeMillis());
ps.setLong(i, jobId);
ps.executeUpdate();
}
}
private void updateJobWithExtra(Connection c, long jobId, String status, String setFragment, Binder binder) throws SQLException {
String statusSql = status == null ? "" : ", status=?";
String sql = "UPDATE archive_publish_job SET " + setFragment + statusSql + ", updated_at_ms=? WHERE id=?";
try (PreparedStatement ps = c.prepareStatement(sql)) {
binder.bind(ps);
int count = countParameters(setFragment);
int i = count + 1;
if (status != null) ps.setString(i++, status);
ps.setLong(i++, System.currentTimeMillis());
ps.setLong(i, jobId);
ps.executeUpdate();
}
}
private static int countParameters(String sqlPart) {
int n = 0;
for (int i = 0; i < sqlPart.length(); i++) if (sqlPart.charAt(i) == '?') n++;
return n;
}
private static void bindJobChain(PreparedStatement ps, ArchivePublishJobChain e) throws SQLException {
int i=1;
ps.setLong(i++, e.jobId());
ps.setString(i++, e.blockchainName());
ps.setLong(i++, e.fromSourceBlockNumber());
ps.setLong(i++, e.toSourceBlockNumber());
ps.setBytes(i++, e.previousSourceBlockHash());
ps.setBytes(i++, e.lastSourceBlockHash());
setLongNullable(ps, i++, e.previousArchiveBigBlockNumber());
ps.setBytes(i++, e.previousArchiveBigBlockHash());
setLongNullable(ps, i++, e.previousChunkOffset());
setLongNullable(ps, i++, e.previousChunkSize());
setLongNullable(ps, i++, e.newChunkOffset());
setLongNullable(ps, i, e.newChunkSize());
}
private static ArchiveChainCursor mapCursor(ResultSet rs) throws SQLException {
return new ArchiveChainCursor(
rs.getString("blockchain_name"),
rs.getLong("last_archived_source_block_number"),
rs.getBytes("last_archived_source_block_hash"),
rs.getLong("last_archive_big_block_number"),
rs.getBytes("last_archive_big_block_hash"),
rs.getLong("last_chunk_offset"),
rs.getLong("last_chunk_size"),
rs.getLong("updated_at_ms")
);
}
private static ArchivePublishJob mapJob(ResultSet rs) throws SQLException {
int conf = rs.getInt("arweave_confirmations");
Integer confObj = rs.wasNull() ? null : conf;
return new ArchivePublishJob(
rs.getLong("id"), rs.getLong("big_block_number"), rs.getString("status"),
rs.getLong("created_at_ms"), rs.getString("local_archive_path"),
rs.getBytes("archive_hash"), rs.getBytes("arweave_tx_id"), confObj,
rs.getString("solana_signature"), rs.getString("error_text"), rs.getLong("updated_at_ms")
);
}
private static ArchivePublishJobChain mapJobChain(ResultSet rs) throws SQLException {
return new ArchivePublishJobChain(
rs.getLong("job_id"), rs.getString("blockchain_name"),
rs.getLong("from_source_block_number"), rs.getLong("to_source_block_number"),
rs.getBytes("previous_source_block_hash"), rs.getBytes("last_source_block_hash"),
getLongNullable(rs, "previous_archive_big_block_number"), rs.getBytes("previous_archive_big_block_hash"),
getLongNullable(rs, "previous_chunk_offset"), getLongNullable(rs, "previous_chunk_size"),
getLongNullable(rs, "new_chunk_offset"), getLongNullable(rs, "new_chunk_size")
);
}
private static Long getLongNullable(ResultSet rs, String name) throws SQLException {
long v = rs.getLong(name);
return rs.wasNull() ? null : v;
}
private static void setLongNullable(PreparedStatement ps, int idx, Long v) throws SQLException {
if (v == null) ps.setNull(idx, Types.BIGINT); else ps.setLong(idx, v);
}
}
@@ -0,0 +1,56 @@
CREATE TABLE IF NOT EXISTS archive_chain_cursor (
blockchain_name TEXT PRIMARY KEY,
last_archived_source_block_number BIGINT NOT NULL,
last_archived_source_block_hash BYTEA NOT NULL,
last_archive_big_block_number BIGINT NOT NULL,
last_archive_big_block_hash BYTEA NOT NULL,
last_chunk_offset BIGINT NOT NULL,
last_chunk_size BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL
);
CREATE TABLE IF NOT EXISTS archive_publish_job (
id BIGSERIAL PRIMARY KEY,
big_block_number BIGINT NOT NULL,
status TEXT NOT NULL,
created_at_ms BIGINT NOT NULL,
local_archive_path TEXT,
archive_hash BYTEA,
arweave_tx_id BYTEA,
arweave_confirmations INTEGER,
solana_signature TEXT,
error_text TEXT,
updated_at_ms BIGINT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_archive_publish_job_status
ON archive_publish_job(status);
CREATE INDEX IF NOT EXISTS idx_archive_publish_job_big_block_number
ON archive_publish_job(big_block_number);
CREATE TABLE IF NOT EXISTS archive_publish_job_chain (
job_id BIGINT NOT NULL REFERENCES archive_publish_job(id) ON DELETE CASCADE,
blockchain_name TEXT NOT NULL,
from_source_block_number BIGINT NOT NULL,
to_source_block_number BIGINT NOT NULL,
previous_source_block_hash BYTEA NOT NULL,
last_source_block_hash BYTEA NOT NULL,
previous_archive_big_block_number BIGINT,
previous_archive_big_block_hash BYTEA,
previous_chunk_offset BIGINT,
previous_chunk_size BIGINT,
new_chunk_offset BIGINT,
new_chunk_size BIGINT,
PRIMARY KEY (job_id, blockchain_name)
);
ALTER TABLE solana_user_pda_current
ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_current
ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_history
ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_history
ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT '';
UPDATE db_schema_version SET schema_version = 22, updated_at_ms = CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT) WHERE id = 1;
@@ -2028,8 +2028,63 @@ CREATE TABLE IF NOT EXISTS user_notification_seen_state (
PRIMARY KEY (owner_login, category)
);
CREATE TABLE IF NOT EXISTS archive_chain_cursor (
blockchain_name TEXT PRIMARY KEY,
last_archived_source_block_number BIGINT NOT NULL,
last_archived_source_block_hash BYTEA NOT NULL,
last_archive_big_block_number BIGINT NOT NULL,
last_archive_big_block_hash BYTEA NOT NULL,
last_chunk_offset BIGINT NOT NULL,
last_chunk_size BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL
);
CREATE TABLE IF NOT EXISTS archive_publish_job (
id BIGSERIAL PRIMARY KEY,
big_block_number BIGINT NOT NULL,
status TEXT NOT NULL,
created_at_ms BIGINT NOT NULL,
local_archive_path TEXT,
archive_hash BYTEA,
arweave_tx_id BYTEA,
arweave_confirmations INTEGER,
solana_signature TEXT,
error_text TEXT,
updated_at_ms BIGINT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_archive_publish_job_status
ON archive_publish_job(status);
CREATE INDEX IF NOT EXISTS idx_archive_publish_job_big_block_number
ON archive_publish_job(big_block_number);
CREATE TABLE IF NOT EXISTS archive_publish_job_chain (
job_id BIGINT NOT NULL REFERENCES archive_publish_job(id) ON DELETE CASCADE,
blockchain_name TEXT NOT NULL,
from_source_block_number BIGINT NOT NULL,
to_source_block_number BIGINT NOT NULL,
previous_source_block_hash BYTEA NOT NULL,
last_source_block_hash BYTEA NOT NULL,
previous_archive_big_block_number BIGINT,
previous_archive_big_block_hash BYTEA,
previous_chunk_offset BIGINT,
previous_chunk_size BIGINT,
new_chunk_offset BIGINT,
new_chunk_size BIGINT,
PRIMARY KEY (job_id, blockchain_name)
);
ALTER TABLE solana_user_pda_current
ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_current
ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_history
ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT '';
ALTER TABLE solana_user_pda_history
ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT '';
INSERT INTO db_schema_version(id,schema_version,updated_at_ms)
VALUES(1,21,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
VALUES(1,22,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
ON CONFLICT(id) DO UPDATE SET schema_version=EXCLUDED.schema_version, updated_at_ms=EXCLUDED.updated_at_ms;
COMMIT;
@@ -46,11 +46,7 @@ import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.Net_AddBlock_R
import server.logic.ws_protocol.JSON.handlers.blockchain.entyties.Net_GetBlockchainBlock_Request;
import server.logic.ws_protocol.JSON.handlers.tempToTest.Net_GetUser_Handler;
import server.logic.ws_protocol.JSON.handlers.tempToTest.Net_TestGetFreeAvatarQuota_Handler;
import server.logic.ws_protocol.JSON.handlers.tempToTest.Net_TestUploadFreeAvatar_Handler;
import server.logic.ws_protocol.JSON.handlers.tempToTest.entyties.Net_GetUser_Request;
import server.logic.ws_protocol.JSON.handlers.tempToTest.entyties.Net_TestGetFreeAvatarQuota_Request;
import server.logic.ws_protocol.JSON.handlers.tempToTest.entyties.Net_TestUploadFreeAvatar_Request;
// --- NEW: SearchUsers ---
import server.logic.ws_protocol.JSON.handlers.tempToTest.Net_SearchUsers_Handler;
@@ -165,8 +161,6 @@ public final class JsonHandlerRegistry {
private static final Map<String, JsonMessageHandler> HANDLERS = Map.ofEntries(
Map.entry("GetUser", new Net_GetUser_Handler()),
Map.entry("SearchUsers", new Net_SearchUsers_Handler()),
Map.entry("TestGetFreeAvatarQuota", new Net_TestGetFreeAvatarQuota_Handler()),
Map.entry("TestUploadFreeAvatar", new Net_TestUploadFreeAvatar_Handler()),
// --- auth ---
Map.entry("ResolveLoginForAuth", new Net_ResolveLoginForAuth_Handler()),
@@ -259,8 +253,6 @@ public final class JsonHandlerRegistry {
private static final Map<String, Class<? extends Net_Request>> REQUEST_TYPES = Map.ofEntries(
Map.entry("GetUser", Net_GetUser_Request.class),
Map.entry("SearchUsers", Net_SearchUsers_Request.class),
Map.entry("TestGetFreeAvatarQuota", Net_TestGetFreeAvatarQuota_Request.class),
Map.entry("TestUploadFreeAvatar", Net_TestUploadFreeAvatar_Request.class),
// --- auth ---
Map.entry("ResolveLoginForAuth", Net_ResolveLoginForAuth_Request.class),
@@ -59,6 +59,9 @@ public final class ShineUsersCodec {
private static final int BLOCK_TYPE_TRUSTED_STATE =
70;
private static final int BLOCK_TYPE_ARCHIVE_HEAD =
100;
private static final int BLOCK_VERSION_0 =
0;
@@ -306,6 +309,12 @@ public final class ShineUsersCodec {
int trustedCount =
0;
String archiveHeadTxId =
"";
String archiveHeadHash =
"";
for (int i = 0; i < blocksCount; i++) {
int blockType =
@@ -455,6 +464,13 @@ public final class ShineUsersCodec {
trustedCount =
reader.readU8();
case BLOCK_TYPE_ARCHIVE_HEAD -> {
archiveHeadTxId =
Base64.getUrlEncoder().withoutPadding().encodeToString(reader.readFixed(32));
archiveHeadHash =
toHex(reader.readFixed(32));
}
default ->
throw new IllegalArgumentException(
"Unsupported block type: " + blockType
@@ -484,6 +500,8 @@ public final class ShineUsersCodec {
lastBlockHash,
lastBlockSignature,
arweaveTxId,
archiveHeadTxId,
archiveHeadHash,
isServer,
addressFormatType,
addressFormatVersion,
@@ -520,7 +538,9 @@ public final class ShineUsersCodec {
mutation.createdAtMs(),
mutation.createdAtMs(),
toHex(ZERO_HASH),
paidLimitBytes
paidLimitBytes,
mutation.fields().archiveHeadSupplied() ? mutation.fields().archiveHeadTxId() : "",
mutation.fields().archiveHeadSupplied() ? mutation.fields().archiveHeadHash() : ""
);
}
@@ -535,6 +555,13 @@ public final class ShineUsersCodec {
previous.paidLimitBytes()
+ mutation.additionalLimit();
String archiveTx = mutation.fields().archiveHeadSupplied()
? mutation.fields().archiveHeadTxId()
: previous.archiveHeadTxId();
String archiveHash = mutation.fields().archiveHeadSupplied()
? mutation.fields().archiveHeadHash()
: previous.archiveHeadHash();
return buildSnapshot(
mutation,
mutation.version(),
@@ -543,7 +570,9 @@ public final class ShineUsersCodec {
mutation.createdAtMs(),
mutation.updatedAtMs(),
toHex(mutation.prevHash()),
paidLimitBytes
paidLimitBytes,
archiveTx,
archiveHash
);
}
@@ -617,10 +646,13 @@ public final class ShineUsersCodec {
pushFixed(out, prevHash);
pushStringU8(out, loginBytes);
boolean hasArchiveHead =
snapshot.archiveHeadTxId() != null
&& !snapshot.archiveHeadTxId().isBlank();
int blocksCount =
snapshot.isServer()
? 8
: 7;
(snapshot.isServer() ? 8 : 7)
+ (hasArchiveHead ? 1 : 0);
out.add((byte) blocksCount);
@@ -695,6 +727,18 @@ public final class ShineUsersCodec {
out.add((byte) BLOCK_VERSION_0);
out.add((byte) snapshot.trustedCount());
if (hasArchiveHead) {
byte[] txId = Base64.getUrlDecoder().decode(snapshot.archiveHeadTxId());
byte[] archiveHash = fromHex(snapshot.archiveHeadHash());
if (txId.length != 32 || archiveHash.length != 32) {
throw new IllegalArgumentException("Archive head должен содержать txId/hash по 32 байта");
}
out.add((byte) BLOCK_TYPE_ARCHIVE_HEAD);
out.add((byte) BLOCK_VERSION_0);
pushFixed(out, txId);
pushFixed(out, archiveHash);
}
int recordLength =
out.size() + 64;
@@ -871,6 +915,8 @@ public final class ShineUsersCodec {
UserFields fields =
parseFields(reader);
fields = parseOptionalArchiveExtension(reader, fields);
String recordSignature =
Base58Util.encode(
reader.readFixed(64)
@@ -1008,7 +1054,35 @@ public final class ShineUsersCodec {
List.copyOf(accessServers),
sessionsMode,
List.copyOf(sessions),
trustedCount
trustedCount,
false,
"",
""
);
}
private static UserFields parseOptionalArchiveExtension(Reader reader, UserFields fields) {
if (reader.remaining() == 64) {
return fields;
}
if (reader.remaining() < 65) {
throw new IllegalArgumentException("Некорректная archive extension в update_user_pda");
}
int present = reader.readU8();
String tx = "";
String hash = "";
if (present == 1) {
tx = Base64.getUrlEncoder().withoutPadding().encodeToString(reader.readFixed(32));
hash = toHex(reader.readFixed(32));
} else if (present != 0) {
throw new IllegalArgumentException("Некорректный archive head marker: " + present);
}
return new UserFields(
fields.clientKey(), fields.blockchainKey(), fields.blockchainName(), fields.usedBytes(),
fields.lastBlockNumber(), fields.lastBlockHash(), fields.lastBlockSignature(), fields.arweaveTxId(),
fields.isServer(), fields.addressFormatType(), fields.addressFormatVersion(), fields.serverAddress(),
fields.syncServers(), fields.accessServers(), fields.sessionsMode(), fields.sessions(), fields.trustedCount(),
true, tx, hash
);
}
@@ -1020,7 +1094,9 @@ public final class ShineUsersCodec {
long createdAtMs,
long updatedAtMs,
String prevRecordHash,
long paidLimitBytes
long paidLimitBytes,
String archiveHeadTxId,
String archiveHeadHash
) {
UserFields fields =
@@ -1044,6 +1120,8 @@ public final class ShineUsersCodec {
fields.lastBlockHash(),
fields.lastBlockSignature(),
fields.arweaveTxId(),
archiveHeadTxId == null ? "" : archiveHeadTxId,
archiveHeadHash == null ? "" : archiveHeadHash,
fields.isServer(),
fields.addressFormatType(),
fields.addressFormatVersion(),
@@ -1266,6 +1344,10 @@ public final class ShineUsersCodec {
) {
cursor += bytes;
}
private int remaining() {
return data.length - cursor;
}
}
public enum TxKind {
@@ -1327,7 +1409,10 @@ public final class ShineUsersCodec {
List<String> accessServers,
int sessionsMode,
List<UserSessionSnapshot> sessions,
int trustedCount
int trustedCount,
boolean archiveHeadSupplied,
String archiveHeadTxId,
String archiveHeadHash
) {
}
@@ -1389,6 +1474,8 @@ public final class ShineUsersCodec {
String lastBlockHash,
String lastBlockSignature,
String arweaveTxId,
String archiveHeadTxId,
String archiveHeadHash,
boolean isServer,
int addressFormatType,
int addressFormatVersion,
@@ -1424,6 +1511,8 @@ public final class ShineUsersCodec {
lastBlockHash,
lastBlockSignature,
arweaveTxId,
archiveHeadTxId,
archiveHeadHash,
isServer,
addressFormatType,
addressFormatVersion,
@@ -425,13 +425,13 @@ public final class PostgresStorageRepository
"record_number, recovery_key, root_key, client_key, " +
"blockchain_name, blockchain_key, paid_limit_bytes, " +
"used_bytes, last_block_number, last_block_hash, " +
"last_block_signature, arweave_tx_id, is_server, " +
"last_block_signature, arweave_tx_id, archive_head_tx_id, archive_head_hash, is_server, " +
"address_format_type, address_format_version, " +
"server_address, sync_servers_json, access_servers_json, " +
"sessions_mode, sessions_json, trusted_count, " +
"created_at_ms, updated_at_ms, prev_record_hash, " +
"record_signature, raw_data_base64, saved_at_ms" +
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " +
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " +
"ON CONFLICT (pda_address, record_number) DO NOTHING";
try (PreparedStatement statement =
@@ -470,13 +470,13 @@ public final class PostgresStorageRepository
"recovery_key, root_key, client_key, blockchain_name, " +
"blockchain_key, paid_limit_bytes, used_bytes, " +
"last_block_number, last_block_hash, last_block_signature, " +
"arweave_tx_id, is_server, address_format_type, " +
"arweave_tx_id, archive_head_tx_id, archive_head_hash, is_server, address_format_type, " +
"address_format_version, server_address, sync_servers_json, " +
"access_servers_json, sessions_mode, sessions_json, " +
"trusted_count, created_at_ms, updated_at_ms, " +
"prev_record_hash, record_signature, raw_data_base64, " +
"first_seen_at_ms, last_synced_at_ms" +
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " +
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " +
"ON CONFLICT (pda_address) DO UPDATE SET " +
"login = EXCLUDED.login, " +
"normalized_login = EXCLUDED.normalized_login, " +
@@ -494,6 +494,8 @@ public final class PostgresStorageRepository
"last_block_hash = EXCLUDED.last_block_hash, " +
"last_block_signature = EXCLUDED.last_block_signature, " +
"arweave_tx_id = EXCLUDED.arweave_tx_id, " +
"archive_head_tx_id = EXCLUDED.archive_head_tx_id, " +
"archive_head_hash = EXCLUDED.archive_head_hash, " +
"is_server = EXCLUDED.is_server, " +
"address_format_type = EXCLUDED.address_format_type, " +
"address_format_version = EXCLUDED.address_format_version, " +
@@ -536,141 +538,41 @@ public final class PostgresStorageRepository
ShineUsersCodec.UserPdaSnapshot snapshot,
long nowMs
) throws Exception {
statement.setString(
1,
snapshot.lastTxSignature()
);
statement.setLong(
2,
snapshot.slot()
);
statement.setNull(
3,
Types.BIGINT
);
statement.setString(
4,
snapshot.pdaAddress()
);
statement.setString(
5,
snapshot.login()
);
statement.setInt(
6,
snapshot.recordNumber()
);
statement.setString(
7,
snapshot.recoveryKey()
);
statement.setString(
8,
snapshot.rootKey()
);
statement.setString(
9,
snapshot.clientKey()
);
statement.setString(
10,
snapshot.blockchainName()
);
statement.setString(
11,
snapshot.blockchainKey()
);
statement.setLong(
12,
snapshot.paidLimitBytes()
);
statement.setLong(
13,
snapshot.usedBytes()
);
statement.setInt(
14,
snapshot.lastBlockNumber()
);
statement.setString(
15,
snapshot.lastBlockHash()
);
statement.setString(
16,
snapshot.lastBlockSignature()
);
statement.setString(
17,
snapshot.arweaveTxId()
);
statement.setBoolean(
18,
snapshot.isServer()
);
statement.setInt(
19,
snapshot.addressFormatType()
);
statement.setInt(
20,
snapshot.addressFormatVersion()
);
statement.setString(
21,
snapshot.serverAddress()
);
statement.setString(
22,
writeJson(
snapshot.syncServers()
)
);
statement.setString(
23,
writeJson(
snapshot.accessServers()
)
);
statement.setInt(
24,
snapshot.sessionsMode()
);
statement.setString(
25,
writeJson(
snapshot.sessions()
)
);
statement.setInt(
26,
snapshot.trustedCount()
);
statement.setLong(
27,
snapshot.createdAtMs()
);
statement.setLong(
28,
snapshot.updatedAtMs()
);
statement.setString(
29,
snapshot.prevRecordHash()
);
statement.setString(
30,
snapshot.recordSignature()
);
statement.setString(
31,
snapshot.rawDataBase64()
);
statement.setLong(
32,
nowMs
);
int i = 1;
statement.setString(i++, snapshot.lastTxSignature());
statement.setLong(i++, snapshot.slot());
statement.setNull(i++, Types.BIGINT);
statement.setString(i++, snapshot.pdaAddress());
statement.setString(i++, snapshot.login());
statement.setInt(i++, snapshot.recordNumber());
statement.setString(i++, snapshot.recoveryKey());
statement.setString(i++, snapshot.rootKey());
statement.setString(i++, snapshot.clientKey());
statement.setString(i++, snapshot.blockchainName());
statement.setString(i++, snapshot.blockchainKey());
statement.setLong(i++, snapshot.paidLimitBytes());
statement.setLong(i++, snapshot.usedBytes());
statement.setInt(i++, snapshot.lastBlockNumber());
statement.setString(i++, snapshot.lastBlockHash());
statement.setString(i++, snapshot.lastBlockSignature());
statement.setString(i++, snapshot.arweaveTxId());
statement.setString(i++, snapshot.archiveHeadTxId());
statement.setString(i++, snapshot.archiveHeadHash());
statement.setBoolean(i++, snapshot.isServer());
statement.setInt(i++, snapshot.addressFormatType());
statement.setInt(i++, snapshot.addressFormatVersion());
statement.setString(i++, snapshot.serverAddress());
statement.setString(i++, writeJson(snapshot.syncServers()));
statement.setString(i++, writeJson(snapshot.accessServers()));
statement.setInt(i++, snapshot.sessionsMode());
statement.setString(i++, writeJson(snapshot.sessions()));
statement.setInt(i++, snapshot.trustedCount());
statement.setLong(i++, snapshot.createdAtMs());
statement.setLong(i++, snapshot.updatedAtMs());
statement.setString(i++, snapshot.prevRecordHash());
statement.setString(i++, snapshot.recordSignature());
statement.setString(i++, snapshot.rawDataBase64());
statement.setLong(i, nowMs);
}
private void bindCurrentSnapshot(
@@ -678,40 +580,42 @@ public final class PostgresStorageRepository
ShineUsersCodec.UserPdaSnapshot snapshot,
long nowMs
) throws Exception {
statement.setString(1, snapshot.pdaAddress());
statement.setString(2, snapshot.login());
statement.setString(3, normalizeLogin(snapshot.login()));
statement.setInt(4, snapshot.recordNumber());
statement.setLong(5, snapshot.slot());
statement.setString(6, snapshot.lastTxSignature());
statement.setString(7, snapshot.recoveryKey());
statement.setString(8, snapshot.rootKey());
statement.setString(9, snapshot.clientKey());
statement.setString(10, snapshot.blockchainName());
statement.setString(11, snapshot.blockchainKey());
statement.setLong(12, snapshot.paidLimitBytes());
statement.setLong(13, snapshot.usedBytes());
statement.setInt(14, snapshot.lastBlockNumber());
statement.setString(15, snapshot.lastBlockHash());
statement.setString(16, snapshot.lastBlockSignature());
statement.setString(17, snapshot.arweaveTxId());
statement.setBoolean(18, snapshot.isServer());
statement.setInt(19, snapshot.addressFormatType());
statement.setInt(20, snapshot.addressFormatVersion());
statement.setString(21, snapshot.serverAddress());
statement.setString(22, writeJson(snapshot.syncServers()));
statement.setString(23, writeJson(snapshot.accessServers()));
statement.setInt(24, snapshot.sessionsMode());
statement.setString(25, writeJson(snapshot.sessions()));
statement.setInt(26, snapshot.trustedCount());
statement.setLong(27, snapshot.createdAtMs());
statement.setLong(28, snapshot.updatedAtMs());
statement.setString(29, snapshot.prevRecordHash());
statement.setString(30, snapshot.recordSignature());
statement.setString(31, snapshot.rawDataBase64());
statement.setLong(32, nowMs);
statement.setLong(33, nowMs);
int i=1;
statement.setString(i++, snapshot.pdaAddress());
statement.setString(i++, snapshot.login());
statement.setString(i++, normalizeLogin(snapshot.login()));
statement.setInt(i++, snapshot.recordNumber());
statement.setLong(i++, snapshot.slot());
statement.setString(i++, snapshot.lastTxSignature());
statement.setString(i++, snapshot.recoveryKey());
statement.setString(i++, snapshot.rootKey());
statement.setString(i++, snapshot.clientKey());
statement.setString(i++, snapshot.blockchainName());
statement.setString(i++, snapshot.blockchainKey());
statement.setLong(i++, snapshot.paidLimitBytes());
statement.setLong(i++, snapshot.usedBytes());
statement.setInt(i++, snapshot.lastBlockNumber());
statement.setString(i++, snapshot.lastBlockHash());
statement.setString(i++, snapshot.lastBlockSignature());
statement.setString(i++, snapshot.arweaveTxId());
statement.setString(i++, snapshot.archiveHeadTxId());
statement.setString(i++, snapshot.archiveHeadHash());
statement.setBoolean(i++, snapshot.isServer());
statement.setInt(i++, snapshot.addressFormatType());
statement.setInt(i++, snapshot.addressFormatVersion());
statement.setString(i++, snapshot.serverAddress());
statement.setString(i++, writeJson(snapshot.syncServers()));
statement.setString(i++, writeJson(snapshot.accessServers()));
statement.setInt(i++, snapshot.sessionsMode());
statement.setString(i++, writeJson(snapshot.sessions()));
statement.setInt(i++, snapshot.trustedCount());
statement.setLong(i++, snapshot.createdAtMs());
statement.setLong(i++, snapshot.updatedAtMs());
statement.setString(i++, snapshot.prevRecordHash());
statement.setString(i++, snapshot.recordSignature());
statement.setString(i++, snapshot.rawDataBase64());
statement.setLong(i++, nowMs);
statement.setLong(i, nowMs);
}
private String normalizeLogin(String login) {
@@ -763,6 +667,8 @@ public final class PostgresStorageRepository
resultSet.getString("last_block_hash"),
resultSet.getString("last_block_signature"),
resultSet.getString("arweave_tx_id"),
resultSet.getString("archive_head_tx_id"),
resultSet.getString("archive_head_hash"),
resultSet.getBoolean("is_server"),
resultSet.getInt("address_format_type"),
resultSet.getInt("address_format_version"),
@@ -933,6 +839,8 @@ public final class PostgresStorageRepository
"last_block_hash TEXT NOT NULL, " +
"last_block_signature TEXT NOT NULL, " +
"arweave_tx_id TEXT NOT NULL, " +
"archive_head_tx_id TEXT NOT NULL DEFAULT '', " +
"archive_head_hash TEXT NOT NULL DEFAULT '', " +
"is_server BOOLEAN NOT NULL, " +
"address_format_type INTEGER NOT NULL, " +
"address_format_version INTEGER NOT NULL, " +
@@ -962,6 +870,9 @@ public final class PostgresStorageRepository
" OR normalized_login <> LOWER(BTRIM(login))"
);
statement.executeUpdate("ALTER TABLE solana_user_pda_current ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT ''");
statement.executeUpdate("ALTER TABLE solana_user_pda_current ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT ''");
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_user_pda_current_slot " +
"ON solana_user_pda_current(slot)"
@@ -995,6 +906,8 @@ public final class PostgresStorageRepository
"last_block_hash TEXT NOT NULL, " +
"last_block_signature TEXT NOT NULL, " +
"arweave_tx_id TEXT NOT NULL, " +
"archive_head_tx_id TEXT NOT NULL DEFAULT '', " +
"archive_head_hash TEXT NOT NULL DEFAULT '', " +
"is_server BOOLEAN NOT NULL, " +
"address_format_type INTEGER NOT NULL, " +
"address_format_version INTEGER NOT NULL, " +
@@ -1014,6 +927,9 @@ public final class PostgresStorageRepository
")"
);
statement.executeUpdate("ALTER TABLE solana_user_pda_history ADD COLUMN IF NOT EXISTS archive_head_tx_id TEXT NOT NULL DEFAULT ''");
statement.executeUpdate("ALTER TABLE solana_user_pda_history ADD COLUMN IF NOT EXISTS archive_head_hash TEXT NOT NULL DEFAULT ''");
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_user_pda_history_login " +
"ON solana_user_pda_history(login)"
@@ -87,7 +87,7 @@ public final class SolanaPdaUtil {
return false;
}
return Ed25519.validatePublicKeyFull(
return Ed25519.validatePublicKeyPartial(
publicKey,
0
);
@@ -0,0 +1,27 @@
package sync.util;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
class SolanaPdaUtilTest {
@Test
void findProgramAddressMatchesSolanaRuntimeForUserLoginSeed() {
String pda = SolanaPdaUtil.findProgramAddress(
List.of(
"user_login=".getBytes(StandardCharsets.UTF_8),
"server_t2".getBytes(StandardCharsets.UTF_8)
),
"SHiNEPr1APdAgNBteUyBXcNovaHctpSjUu8oH2ZJdN6"
);
assertEquals(
"EJCW7cPXgEMvQAzmhJMZ7pxQxxxFbQYZoJNhcfJpWnz8",
pda
);
}
}
@@ -5,6 +5,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.websocket.server.config.JettyWebSocketServletContainerInitializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import server.archive.ArchivePublisherScheduler;
import server.debug.DebugApiConfigurator;
import server.files.DmFileApiConfigurator;
import server.sync.BlockchainResyncRecoveryOnStartup;
@@ -82,6 +83,9 @@ public final class WsServer {
// ============================================================
PeriodicBlockchainSyncService.startOrLog();
// Опциональная серверная публикация больших SHINE-ARCHIVE блоков.
ArchivePublisherScheduler.startOrLog();
// ============================================================
// 2) Запуск Jetty WS
// ============================================================
@@ -110,7 +114,7 @@ public final class WsServer {
log.info("✅ WS сервер запущен на ws://localhost:{}/ws", port);
ServerConnectionPool.getInstance().startOrLog();
Runtime.getRuntime().addShutdownHook(new Thread(
() -> ServerConnectionPool.getInstance().close(),
() -> { ServerConnectionPool.getInstance().close(); ArchivePublisherScheduler.close(); },
"server-connection-pool-shutdown"));
PeriodicDmDeliveryService.startOrLog();
server.join();
@@ -14,6 +14,34 @@ solana.users.sync.dbUser=
solana.users.sync.dbPassword=
solana.users.sync.pollIntervalSeconds=300
# ------------------------------------------------------------
# SHiNE archive publisher (Arweave + User PDA archive head)
# По умолчанию выключен: большинство серверов только читают/синхронизируют блоки.
# При включении готовые большие файлы сначала сохраняются локально в data/archive,
# затем загружаются в Arweave и только после подтверждения фиксируются в Solana.
# ------------------------------------------------------------
archive.publish.enabled=false
archive.publish.time=00:00
# Пусто = системная timezone сервера. Можно задать, например Europe/Warsaw.
archive.publish.zoneId=
archive.maxFileBytes=4000000000
archive.workDir=data/archive
archive.arweave.gateway=https://arweave.net
archive.arweave.walletJwkPath=
archive.arweave.minConfirmations=1
archive.arweave.confirmPollSeconds=30
archive.arweave.confirmTimeoutMinutes=180
# Для записи archive head используется уже существующий Solana RPC:
# solana.users.sync.rpcUrl (если задан), иначе solana.rpcUrl.
# Отдельного archive.solana.rpcUrl нет.
archive.solana.rootKeyPath=
archive.solana.clientKeyPath=
archive.solana.confirmPollSeconds=5
archive.solana.confirmTimeoutMinutes=30
archive.solana.commitment=finalized
# ------------------------------------------------------------
# Межсерверная синхронизация: как создавать локальную запись пользователя,
# если во время sync пришла чужая цепочка, а у нас такого login ещё нет.