Новая схема работы Arweave через Turbo

This commit is contained in:
AidarKC
2026-09-25 13:19:24 +03:00
parent ef707ec217
commit 408e474130
30 changed files with 544 additions and 241 deletions
@@ -20,7 +20,7 @@ public final class ArweaveBlockPublisherScheduler {
ArweaveBlocksConfig cfg;
try { cfg = ArweaveBlocksConfig.load(); cfg.validatePublisher(); }
catch (Exception e) { log.error("Cannot read/validate Arweave block publisher config", e); return; }
if (!cfg.publishEnabled()) { log.info("Arweave user-block publisher disabled"); return; }
if (!cfg.publishEnabled()) { log.info("Arweave user-block publisher disabled (mode=none)"); return; }
if (!STARTED.compareAndSet(false,true)) return;
try {
ArweaveBlockPublisherService service = new ArweaveBlockPublisherService(cfg);
@@ -29,7 +29,7 @@ public final class ArweaveBlockPublisherScheduler {
});
Runnable task = () -> { try { service.runCycle(); } catch (Exception e) { log.error("Arweave block publish cycle failed", e); } };
executor.scheduleWithFixedDelay(task, 0, cfg.publishIntervalMinutes(), TimeUnit.MINUTES);
log.info("Arweave user-block publisher enabled: interval={}m gateway={}", cfg.publishIntervalMinutes(), cfg.publishGateway());
log.info("Arweave user-block publisher enabled: mode={} interval={}m", cfg.publishMode(), cfg.publishIntervalMinutes());
} catch (Exception e) { STARTED.set(false); log.error("Arweave user-block publisher failed to start", e); }
}
@@ -8,23 +8,56 @@ import shine.db.entities.BlockEntry;
import java.util.ArrayList;
import java.util.List;
/** Publishes locally-created signed user DataItems as one standard ANS-104 bundle. */
/** Publishes locally-created signed user DataItems through the configured transport. */
public final class ArweaveBlockPublisherService {
private static final Logger log = LoggerFactory.getLogger(ArweaveBlockPublisherService.class);
private final ArweaveBlocksConfig cfg;
private final BlocksDAO blocksDAO = BlocksDAO.getInstance();
private final ArweaveL1Uploader uploader;
private final ArweaveL1Uploader arweaveUploader;
private final TurboDataItemUploader turboUploader;
public ArweaveBlockPublisherService(ArweaveBlocksConfig cfg) {
this.cfg = cfg;
this.uploader = new ArweaveL1Uploader(cfg);
this.arweaveUploader = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.ARWEAVE ? new ArweaveL1Uploader(cfg) : null;
this.turboUploader = cfg.publishMode() == ArweaveBlocksConfig.PublishMode.TURBO ? new TurboDataItemUploader(cfg) : null;
}
public int runCycle() throws Exception {
if (cfg.publishMode() == ArweaveBlocksConfig.PublishMode.NONE) return 0;
List<BlockEntry> candidates = blocksDAO.listPendingArweave(cfg.publishMaxItems());
if (candidates.isEmpty()) return 0;
return switch (cfg.publishMode()) {
case TURBO -> publishTurbo(candidates);
case ARWEAVE -> publishDirectArweave(candidates);
case NONE -> 0;
};
}
private int publishTurbo(List<BlockEntry> candidates) throws Exception {
int published = 0;
Exception firstFailure = null;
for (BlockEntry e : candidates) {
byte[] raw = e.getBlockBytes();
byte[] id = e.getDataItemId();
if (raw == null || raw.length == 0 || id == null || id.length != 32) continue;
try {
TurboDataItemUploader.UploadResult result = turboUploader.upload(raw, id);
blocksDAO.markArweavePublished(List.of(id), System.currentTimeMillis());
published++;
log.debug("Turbo published SHiNE DataItem {} chain={} block={}", result.dataItemId(), e.getBchName(), e.getBlockNumber());
} catch (Exception ex) {
if (firstFailure == null) firstFailure = ex;
log.warn("Turbo publish failed: chain={} block={} bytes={} error={}",
e.getBchName(), e.getBlockNumber(), raw.length, ex.getMessage());
}
}
if (published > 0) log.info("Published {} SHiNE test DataItems through Turbo", published);
if (published == 0 && firstFailure != null) throw firstFailure;
return published;
}
private int publishDirectArweave(List<BlockEntry> candidates) throws Exception {
List<byte[]> items = new ArrayList<>();
List<byte[]> ids = new ArrayList<>();
long estimated = 32;
@@ -39,7 +72,9 @@ public final class ArweaveBlockPublisherService {
e.getBchName(), e.getBlockNumber(), raw.length);
continue;
}
items.add(raw); ids.add(id); estimated = next;
items.add(raw);
ids.add(id);
estimated = next;
}
if (items.isEmpty()) return 0;
@@ -47,13 +82,12 @@ public final class ArweaveBlockPublisherService {
List<ArweaveL1Uploader.Tag> rootTags = List.of(
new ArweaveL1Uploader.Tag("Content-Type", "application/octet-stream"),
new ArweaveL1Uploader.Tag("Bundle-Format", "binary"),
new ArweaveL1Uploader.Tag("Bundle-Version", "2.0.0"),
// Deliberately different from App=test5590 so discovery returns child DataItems only.
new ArweaveL1Uploader.Tag("App", "test5590-batch")
new ArweaveL1Uploader.Tag("Bundle-Version", "2.0.0")
);
ArweaveL1Uploader.UploadResult result = uploader.upload(bundle, rootTags);
blocksDAO.markArweavePublished(ids, result.txId(), System.currentTimeMillis());
log.info("Published {} SHiNE test DataItems in root tx {} (bundle={} bytes)", items.size(), result.txId(), bundle.length);
ArweaveL1Uploader.UploadResult result = arweaveUploader.upload(bundle, rootTags);
blocksDAO.markArweavePublished(ids, System.currentTimeMillis());
log.info("Published {} SHiNE test DataItems in direct Arweave root tx {} (bundle={} bytes)",
items.size(), result.txId(), bundle.length);
return items.size();
}
}
@@ -14,7 +14,9 @@ import shine.db.dao.SolanaUserPdaCurrentDAO;
import shine.db.entities.BlockchainStateEntry;
import shine.db.entities.SolanaUserPdaCurrentEntry;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@@ -24,8 +26,9 @@ import java.time.Duration;
import java.util.*;
/**
* Discovers App=test5590 child DataItems, extracts their exact serialized bytes
* from the root ANS-104 bundle and imports them through the normal AddBlock checks.
* Discovers individual App=test5590 DataItems and imports the exact signed ANS-104 bytes.
* The importer is transport-agnostic: a DataItem may have reached Arweave through Turbo
* or inside a direct server-created ANS-104 bundle.
*/
public final class ArweaveBlockSyncService {
private static final Logger log = LoggerFactory.getLogger(ArweaveBlockSyncService.class);
@@ -55,10 +58,11 @@ public final class ArweaveBlockSyncService {
long minHeight = Math.max(cfg.syncStartBlockHeight(), stored);
String cursor = null;
long maxHeight = minHeight;
long lowestRetryHeight = Long.MAX_VALUE;
int discovered = 0;
do {
JsonNode response = graphQlPage(minHeight, cursor, ArweaveBlocksConfig.TEST_TAG_VALUE + "-batch");
JsonNode response = graphQlPage(minHeight, cursor);
JsonNode txs = response.path("data").path("transactions");
if (response.has("errors")) throw new IOException("Arweave GraphQL errors: " + response.path("errors"));
JsonNode edges = txs.path("edges");
@@ -68,22 +72,27 @@ public final class ArweaveBlockSyncService {
for (JsonNode edge : edges) {
nextCursor = edge.path("cursor").asText(null);
JsonNode node = edge.path("node");
String rootTx = node.path("id").asText("").trim();
String dataItemId = node.path("id").asText("").trim();
long height = node.path("block").path("height").asLong(-1L);
if (rootTx.isBlank() || height < 0) continue;
if (dataItemId.isBlank() || height < 0) continue;
maxHeight = Math.max(maxHeight, height);
byte[] bundle = downloadRootBundle(rootTx);
for (Ans104Bundle.Entry entry : Ans104Bundle.read(bundle, Math.max(cfg.publishMaxItems() * 4, 100_000))) {
byte[] id32 = entry.id32();
if (id32 == null || id32.length != 32 || blocksDAO.existsByDataItemId(id32) || importDAO.exists(id32)) {
continue;
}
Ans104DataItem parsed = new Ans104DataItem(entry.rawDataItem());
if (!parsed.hasTag(ArweaveBlocksConfig.TEST_TAG_NAME, ArweaveBlocksConfig.TEST_TAG_VALUE)) continue;
if (!parsed.verifySignature()) throw new IOException("Bad ANS-104 signature in root bundle " + rootTx);
importDAO.enqueueIfMissing(id32, rootTx, height, entry.rawDataItem(), System.currentTimeMillis());
discovered++;
byte[] id32;
try {
id32 = B64URL.decode(dataItemId);
if (id32.length != 32) throw new IllegalArgumentException("id length=" + id32.length);
} catch (Exception e) {
log.warn("Ignoring malformed Arweave DataItem id {}: {}", dataItemId, e.getMessage());
continue;
}
if (blocksDAO.existsByDataItemId(id32) || importDAO.exists(id32)) continue;
try {
byte[] rawDataItem = downloadSignedDataItem(dataItemId);
if (importDAO.enqueueIfMissing(id32, height, rawDataItem, System.currentTimeMillis())) discovered++;
} catch (Exception e) {
lowestRetryHeight = Math.min(lowestRetryHeight, height);
log.warn("Cannot retrieve signed DataItem {} at height {} yet: {}", dataItemId, height, e.getMessage());
}
}
boolean hasNext = txs.path("pageInfo").path("hasNextPage").asBoolean(false);
@@ -91,9 +100,12 @@ public final class ArweaveBlockSyncService {
if (hasNext && (cursor == null || cursor.isBlank())) throw new IOException("GraphQL hasNextPage without cursor");
} while (cursor != null);
// Keep one-height overlap: the next query includes this height and deduplicates IDs.
importDAO.setLastBlockHeight(maxHeight, System.currentTimeMillis());
if (discovered > 0) log.info("Arweave discovery queued {} new SHiNE test DataItems through height {}", discovered, maxHeight);
// Keep inclusive overlap. If a gateway has indexed GraphQL before offsets, do not advance past that item.
long checkpoint = lowestRetryHeight == Long.MAX_VALUE ? maxHeight : Math.min(maxHeight, lowestRetryHeight);
importDAO.setLastBlockHeight(checkpoint, System.currentTimeMillis());
if (discovered > 0) {
log.info("Arweave discovery queued {} new SHiNE test DataItems; checkpoint={}", discovered, checkpoint);
}
}
private void drainQueue() throws Exception {
@@ -106,13 +118,12 @@ public final class ArweaveBlockSyncService {
if (pending.isEmpty()) break;
for (ArweaveBlockImportDAO.QueueItem q : pending) {
Ans104DataItem item;
BchBlockEntry block;
try {
item = new Ans104DataItem(q.rawDataItem());
if (!Arrays.equals(item.id32(), q.dataItemId())) throw new IllegalArgumentException("data_item_id mismatch");
if (!item.hasTag(ArweaveBlocksConfig.TEST_TAG_NAME, ArweaveBlocksConfig.TEST_TAG_VALUE)) throw new IllegalArgumentException("bad App tag");
if (!item.verifySignature()) throw new IllegalArgumentException("bad ANS-104 signature");
block = new BchBlockEntry(q.rawDataItem());
new BchBlockEntry(q.rawDataItem());
} catch (Exception e) {
importDAO.reject(q.dataItemId(), "invalid_data_item: " + e.getMessage(), System.currentTimeMillis());
continue;
@@ -154,9 +165,12 @@ public final class ArweaveBlockSyncService {
stateDAO.insertIfMissing(s);
}
private JsonNode graphQlPage(long minHeight, String cursor, String appTagValue) throws Exception {
private JsonNode graphQlPage(long minHeight, String cursor) throws Exception {
String after = cursor == null ? "null" : "\"" + escapeGraphQl(cursor) + "\"";
String query = "query { transactions(tags:[{name:\"App\",values:[\"" + escapeGraphQl(appTagValue) + "\"]}], block:{min:" + minHeight + "}, first:" + cfg.syncPageSize() + ", after:" + after + ", sort:HEIGHT_ASC) { pageInfo { hasNextPage } edges { cursor node { id block { height } } } } }";
String query = "query { transactions(tags:[{name:\"" + ArweaveBlocksConfig.TEST_TAG_NAME + "\",values:[\""
+ ArweaveBlocksConfig.TEST_TAG_VALUE + "\"]}], block:{min:" + minHeight + "}, first:"
+ cfg.syncPageSize() + ", after:" + after
+ ", sort:HEIGHT_ASC) { pageInfo { hasNextPage } edges { cursor node { id block { height } } } } }";
String body = MAPPER.writeValueAsString(Map.of("query", query));
HttpRequest req = HttpRequest.newBuilder(URI.create(trim(cfg.syncGateway()) + "/graphql"))
.timeout(Duration.ofSeconds(60)).header("Content-Type","application/json").header("Accept","application/json")
@@ -166,17 +180,73 @@ public final class ArweaveBlockSyncService {
return MAPPER.readTree(resp.body());
}
private byte[] downloadRootBundle(String txId) throws Exception {
HttpRequest req = HttpRequest.newBuilder(URI.create(trim(cfg.syncGateway()) + "/" + txId))
.timeout(Duration.ofMinutes(5)).GET().build();
HttpResponse<byte[]> resp = http.send(req, HttpResponse.BodyHandlers.ofByteArray());
if (resp.statusCode() < 200 || resp.statusCode() >= 300) throw new IOException("Arweave root HTTP " + resp.statusCode() + " tx=" + txId);
byte[] body = resp.body();
if (body == null || body.length == 0) throw new IOException("Empty Arweave root bundle " + txId);
if (body.length > cfg.syncMaxRootBundleBytes()) throw new IOException("Root bundle exceeds syncMaxRootBundleBytes: " + body.length);
return body;
/**
* Gateways normally expose only the payload at /{dataItemId}. SHiNE needs the complete signed
* DataItem, so obtain its exact offset/size inside the root L1 transaction and range-read it.
*/
private byte[] downloadSignedDataItem(String dataItemId) throws Exception {
JsonNode offsets = getOffsets(dataItemId);
String rootTxId = offsets.path("rootTxId").asText("").trim();
long rootOffset = offsets.path("rootOffset").asLong(-1L);
long size = offsets.path("size").asLong(-1L);
if (rootTxId.isBlank() || rootOffset < 0 || size <= 0) {
throw new IOException("Bad AR.IO offsets for " + dataItemId + ": " + offsets);
}
if (size > cfg.syncMaxDataItemBytes()) {
throw new IOException("DataItem exceeds syncMaxDataItemBytes: " + size);
}
if (rootOffset > Long.MAX_VALUE - size) throw new IOException("DataItem offset overflow");
long endInclusive = rootOffset + size - 1;
HttpRequest req = HttpRequest.newBuilder(URI.create(trim(cfg.syncGateway()) + "/raw/" + rootTxId))
.timeout(Duration.ofMinutes(2))
.header("Range", "bytes=" + rootOffset + "-" + endInclusive)
.header("Accept", "application/octet-stream")
.GET().build();
HttpResponse<InputStream> resp = http.send(req, HttpResponse.BodyHandlers.ofInputStream());
try (InputStream in = resp.body()) {
if (resp.statusCode() != 206) {
throw new IOException("Gateway ignored root range for DataItem " + dataItemId + ": HTTP " + resp.statusCode());
}
byte[] bytes = readExactlyBounded(in, (int) size);
if (bytes.length != size) throw new IOException("Truncated DataItem range: expected=" + size + " got=" + bytes.length);
Ans104DataItem parsed = new Ans104DataItem(bytes);
byte[] expectedId = B64URL.decode(dataItemId);
if (!Arrays.equals(parsed.id32(), expectedId)) {
throw new IOException("Range returned another ANS-104 DataItem for " + dataItemId);
}
return bytes;
}
}
private JsonNode getOffsets(String dataItemId) throws Exception {
HttpRequest req = HttpRequest.newBuilder(URI.create(trim(cfg.syncGateway()) + "/ar-io/offsets/" + dataItemId))
.timeout(Duration.ofSeconds(30)).header("Accept","application/json").GET().build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() == 404) throw new IOException("AR.IO offsets not indexed yet");
if (resp.statusCode() < 200 || resp.statusCode() >= 300) {
throw new IOException("AR.IO offsets HTTP " + resp.statusCode() + ": " + safe(resp.body()));
}
return MAPPER.readTree(resp.body());
}
private static byte[] readExactlyBounded(InputStream in, int expected) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(expected);
byte[] buf = new byte[Math.min(64 * 1024, Math.max(1024, expected))];
int remaining = expected;
while (remaining > 0) {
int n = in.read(buf, 0, Math.min(buf.length, remaining));
if (n < 0) break;
out.write(buf, 0, n);
remaining -= n;
}
return out.toByteArray();
}
private static String safe(String value) {
String v = value == null ? "" : value.replace('\n',' ').replace('\r',' ').trim();
return v.length() <= 500 ? v : v.substring(0, 500);
}
private static String trim(String s){return String.valueOf(s==null?"":s).trim().replaceAll("/+$","");}
private static String escapeGraphQl(String s){return s.replace("\\","\\\\").replace("\"","\\\"");}
}
@@ -1,12 +1,14 @@
package server.archive;
import blockchain.Ans104DataItem;
import utils.config.AppConfig;
import java.nio.file.Path;
import java.util.Locale;
/** Configuration of the new per-user-block ANS-104 Arweave transport. */
/** Configuration of per-user-block ANS-104 Arweave/Turbo transport. */
public record ArweaveBlocksConfig(
boolean publishEnabled,
PublishMode publishMode,
int publishIntervalMinutes,
int publishMaxItems,
long publishMaxBundleBytes,
@@ -15,22 +17,44 @@ public record ArweaveBlocksConfig(
int minConfirmations,
int confirmPollSeconds,
int confirmTimeoutMinutes,
String turboUploadUrl,
String turboPaidByAddress,
Path turboWalletJwkPath,
boolean syncEnabled,
int syncIntervalMinutes,
int syncPageSize,
int syncQueueBatchSize,
long syncStartBlockHeight,
long syncMaxRootBundleBytes,
long syncMaxDataItemBytes,
String syncGateway
) {
public enum PublishMode {
TURBO,
ARWEAVE,
NONE;
static PublishMode parse(String value) {
String normalized = value == null ? "none" : value.trim().toLowerCase(Locale.ROOT);
return switch (normalized) {
case "turbo" -> TURBO;
case "arweave" -> ARWEAVE;
case "none", "" -> NONE;
default -> throw new IllegalArgumentException(
"arweave.blocks.publish.mode must be turbo, arweave or none; got: " + value);
};
}
}
public static final String TEST_TAG_NAME = "App";
public static final String TEST_TAG_VALUE = "test5590";
public static final String CHANNEL_TAG_NAME = "c";
public static final String CHANNEL_TAG_NAME = "c_test5590";
public static ArweaveBlocksConfig load() {
AppConfig c = AppConfig.getInstance();
long maxDataItemBytes = parseLong(c.getParam("arweave.blocks.sync.maxDataItemBytes"), Ans104DataItem.MAX_DATA_ITEM_BYTES);
if (maxDataItemBytes > Ans104DataItem.MAX_DATA_ITEM_BYTES) maxDataItemBytes = Ans104DataItem.MAX_DATA_ITEM_BYTES;
return new ArweaveBlocksConfig(
c.getBoolean("arweave.blocks.publish.enabled", false),
PublishMode.parse(c.getParam("arweave.blocks.publish.mode")),
positive(c.getInt("arweave.blocks.publish.intervalMinutes", 15), "publish.intervalMinutes"),
positive(c.getInt("arweave.blocks.publish.maxItems", 10_000), "publish.maxItems"),
positiveLong(parseLong(c.getParam("arweave.blocks.publish.maxBundleBytes"), 128L * 1024 * 1024), "publish.maxBundleBytes"),
@@ -39,23 +63,32 @@ public record ArweaveBlocksConfig(
nonNegative(c.getInt("arweave.blocks.publish.minConfirmations", 0), "publish.minConfirmations"),
positive(c.getInt("arweave.blocks.publish.confirmPollSeconds", 30), "publish.confirmPollSeconds"),
positive(c.getInt("arweave.blocks.publish.confirmTimeoutMinutes", 180), "publish.confirmTimeoutMinutes"),
orDefault(c.getParam("arweave.blocks.publish.turbo.uploadUrl"), "https://turbo.ardrive.io/tx"),
blankToNull(c.getParam("arweave.blocks.publish.turbo.paidByAddress")),
optionalPath(c.getParam("arweave.blocks.publish.turbo.walletJwkPath")),
c.getBoolean("arweave.blocks.sync.enabled", false),
positive(c.getInt("arweave.blocks.sync.intervalMinutes", 15), "sync.intervalMinutes"),
clamp(c.getInt("arweave.blocks.sync.pageSize", 100), 1, 100),
positive(c.getInt("arweave.blocks.sync.queueBatchSize", 10_000), "sync.queueBatchSize"),
nonNegativeLong(parseLong(c.getParam("arweave.blocks.sync.startBlockHeight"), 0L), "sync.startBlockHeight"),
positiveLong(parseLong(c.getParam("arweave.blocks.sync.maxRootBundleBytes"), 256L * 1024 * 1024), "sync.maxRootBundleBytes"),
positiveLong(maxDataItemBytes, "sync.maxDataItemBytes"),
orDefault(c.getParam("arweave.blocks.sync.gateway"), "https://turbo-gateway.com")
);
}
public boolean publishEnabled() { return publishMode != PublishMode.NONE; }
public void validatePublisher() {
if (publishEnabled && walletJwkPath == null) {
throw new IllegalArgumentException("arweave.blocks.publish.walletJwkPath is required when publisher is enabled");
if (publishMode == PublishMode.ARWEAVE && walletJwkPath == null) {
throw new IllegalArgumentException("arweave.blocks.publish.walletJwkPath is required for mode=arweave");
}
if (publishMode == PublishMode.TURBO && (turboUploadUrl == null || turboUploadUrl.isBlank())) {
throw new IllegalArgumentException("arweave.blocks.publish.turbo.uploadUrl is required for mode=turbo");
}
}
private static String orDefault(String v, String d) { return v == null || v.isBlank() ? d : v.trim(); }
private static String blankToNull(String v) { return v == null || v.isBlank() ? null : v.trim(); }
private static Path optionalPath(String v) { return v == null || v.isBlank() ? null : Path.of(v.trim()); }
private static long parseLong(String v, long d) { return v == null || v.isBlank() ? d : Long.parseLong(v.trim()); }
private static int positive(int v,String n){if(v<=0)throw new IllegalArgumentException(n+" must be >0");return v;}
@@ -0,0 +1,106 @@
package server.archive;
import blockchain.Ans104DataItem;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.time.Duration;
import java.util.Arrays;
import java.util.Base64;
import java.util.Objects;
/** Uploads an already user-signed ANS-104 DataItem to Turbo without modifying it. */
public final class TurboDataItemUploader {
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final Base64.Encoder B64URL = Base64.getUrlEncoder().withoutPadding();
private static final Base64.Decoder B64URL_DECODER = Base64.getUrlDecoder();
public record UploadResult(String dataItemId, String owner) {}
private final ArweaveBlocksConfig cfg;
private final HttpClient http = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(20))
.followRedirects(HttpClient.Redirect.NORMAL)
.build();
private volatile String resolvedPaidByAddress;
public TurboDataItemUploader(ArweaveBlocksConfig cfg) {
this.cfg = Objects.requireNonNull(cfg);
}
public UploadResult upload(byte[] rawDataItem, byte[] expectedId32) throws Exception {
if (rawDataItem == null || rawDataItem.length == 0) throw new IllegalArgumentException("Turbo DataItem is empty");
Ans104DataItem item = new Ans104DataItem(rawDataItem);
if (!item.verifySignature()) throw new IllegalArgumentException("Turbo DataItem has bad ANS-104 signature");
if (expectedId32 != null && !Arrays.equals(item.id32(), expectedId32)) {
throw new IllegalArgumentException("Turbo DataItem id does not match blocks.data_item_id");
}
String expectedId = B64URL.encodeToString(item.id32());
HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(cfg.turboUploadUrl()))
.timeout(Duration.ofMinutes(2))
.header("Content-Type", "application/octet-stream")
.header("Accept", "application/json")
.POST(HttpRequest.BodyPublishers.ofByteArray(rawDataItem));
String paidBy = paidByAddress();
if (paidBy != null) request.header("x-paid-by", paidBy);
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
String body = response.body() == null ? "" : response.body().trim();
if (response.statusCode() == 409 && body.toLowerCase().contains("data item exists")) {
return new UploadResult(expectedId, B64URL.encodeToString(item.owner32()));
}
if (response.statusCode() < 200 || response.statusCode() >= 300) {
String hint = response.statusCode() == 402
? " (Turbo payment required: check server credits / Credit Share Approval / x-paid-by)"
: "";
throw new IOException("Turbo HTTP " + response.statusCode() + hint + ": " + safe(body));
}
if (body.isBlank()) return new UploadResult(expectedId, B64URL.encodeToString(item.owner32()));
JsonNode json;
try { json = MAPPER.readTree(body); }
catch (Exception ignored) { return new UploadResult(expectedId, B64URL.encodeToString(item.owner32())); }
String returnedId = json.path("id").asText("").trim();
if (!returnedId.isBlank() && !returnedId.equals(expectedId)) {
throw new IOException("Turbo returned another DataItem id: expected=" + expectedId + " got=" + returnedId);
}
return new UploadResult(expectedId, json.path("owner").asText(""));
}
/**
* x-paid-by contains the payer's public native address, never the private key.
* A configured Arweave JWK is used only to derive that public address.
*/
private String paidByAddress() throws Exception {
if (resolvedPaidByAddress != null) return resolvedPaidByAddress.isBlank() ? null : resolvedPaidByAddress;
synchronized (this) {
if (resolvedPaidByAddress != null) return resolvedPaidByAddress.isBlank() ? null : resolvedPaidByAddress;
String explicit = cfg.turboPaidByAddress();
if (explicit != null && !explicit.isBlank()) return resolvedPaidByAddress = explicit.trim();
if (cfg.turboWalletJwkPath() == null) {
resolvedPaidByAddress = "";
return null;
}
JsonNode jwk = MAPPER.readTree(Files.readString(cfg.turboWalletJwkPath(), StandardCharsets.UTF_8));
String modulus = jwk.path("n").asText("").trim();
if (modulus.isBlank()) throw new IllegalStateException("Turbo payer JWK missing n");
byte[] owner = B64URL_DECODER.decode(modulus);
resolvedPaidByAddress = B64URL.encodeToString(MessageDigest.getInstance("SHA-256").digest(owner));
return resolvedPaidByAddress;
}
}
private static String safe(String value) {
String v = value == null ? "" : value.replace('\n',' ').replace('\r',' ').trim();
return v.length() <= 500 ? v : v.substring(0, 500);
}
}
@@ -16,7 +16,7 @@ final class Ans104DataItemTest {
byte[] data = "frame-v1-test".getBytes(StandardCharsets.UTF_8);
List<Ans104DataItem.Tag> tags = List.of(
new Ans104DataItem.Tag("App", "test5590"),
new Ans104DataItem.Tag("c", "books")
new Ans104DataItem.Tag("c_test5590", "books")
);
byte[] message = Ans104DataItem.buildSigningMessage(owner, tags, data);
@@ -28,7 +28,7 @@ final class Ans104DataItemTest {
assertArrayEquals(owner, parsed.owner32());
assertArrayEquals(data, parsed.data());
assertTrue(parsed.hasTag("App", "test5590"));
assertEquals("books", parsed.tagValue("c"));
assertEquals("books", parsed.tagValue("c_test5590"));
assertTrue(parsed.verifySignature());
assertEquals(32, parsed.id32().length);
}
@@ -51,11 +51,11 @@ final class Ans104DataItemTest {
byte[] data = "frame-v1-test".getBytes(StandardCharsets.UTF_8);
List<Ans104DataItem.Tag> tags = List.of(
new Ans104DataItem.Tag("App", "test5590"),
new Ans104DataItem.Tag("c", "books")
new Ans104DataItem.Tag("c_test5590", "books")
);
byte[] actual = Ans104DataItem.buildSigningMessage(owner, tags, data);
byte[] expected = hex("7e67d0debce103606d697a1e3785130ca20cb89cd8a06f9a65b98b2d2427eaf411fcf7d482da268e44f3ac25c57c3cb9");
byte[] expected = hex("1abe0371d12268b34be32ca5ebf3d3d2189f9d311d9004c142538ca8a4312faa329d1a4873df72dd01b632a2ecce6b87");
assertArrayEquals(expected, actual);
}
@@ -41,6 +41,7 @@ public final class DatabaseInitializer {
public static final int SCHEMA_VERSION_22 = 22;
public static final int SCHEMA_VERSION_23 = 23;
public static final int SCHEMA_VERSION_24 = 24;
public static final int SCHEMA_VERSION_25 = 25;
public static final String POSTGRES_SCHEMA_RESOURCE = "postgres/schema_v1.sql";
public static final String POSTGRES_MIGRATION_V2_RESOURCE = "postgres/migration_v2.sql";
public static final String POSTGRES_MIGRATION_V3_RESOURCE = "postgres/migration_v3.sql";
@@ -65,6 +66,7 @@ public final class DatabaseInitializer {
public static final String POSTGRES_MIGRATION_V22_RESOURCE = "postgres/migration_v22.sql";
public static final String POSTGRES_MIGRATION_V23_RESOURCE = "postgres/migration_v23.sql";
public static final String POSTGRES_MIGRATION_V24_RESOURCE = "postgres/migration_v24.sql";
public static final String POSTGRES_MIGRATION_V25_RESOURCE = "postgres/migration_v25.sql";
private DatabaseInitializer() {}
@@ -230,6 +232,10 @@ public final class DatabaseInitializer {
runSqlScript(conn, POSTGRES_MIGRATION_V24_RESOURCE);
currentVersion = SCHEMA_VERSION_24;
}
if (currentVersion < SCHEMA_VERSION_25) {
runSqlScript(conn, POSTGRES_MIGRATION_V25_RESOURCE);
currentVersion = SCHEMA_VERSION_25;
}
}
}
@@ -6,12 +6,12 @@ import java.sql.*;
import java.util.ArrayList;
import java.util.List;
/** Persistent discovery/import queue for ANS-104 SHiNE blocks found through Arweave. */
/** Persistent discovery/import queue for individual ANS-104 SHiNE DataItems found through Arweave. */
public final class ArweaveBlockImportDAO {
public static final String STATUS_PENDING = "PENDING";
public static final String STATUS_REJECTED = "REJECTED";
public record QueueItem(byte[] dataItemId, String rootTxId, long blockHeight,
public record QueueItem(byte[] dataItemId, long blockHeight,
byte[] rawDataItem, String status, String lastError,
long firstSeenAtMs, long updatedAtMs) {}
@@ -51,22 +51,21 @@ public final class ArweaveBlockImportDAO {
}
/** Insert once. Existing IDs (including REJECTED) are deliberately not re-enqueued. */
public boolean enqueueIfMissing(byte[] dataItemId, String rootTxId, long blockHeight, byte[] rawDataItem, long nowMs)
public boolean enqueueIfMissing(byte[] dataItemId, long blockHeight, byte[] rawDataItem, long nowMs)
throws SQLException {
String sql = """
INSERT INTO arweave_block_import_queue(
data_item_id,root_tx_id,block_height,raw_data_item,status,last_error,first_seen_at_ms,updated_at_ms
) VALUES(?,?,?,?,?,'',?,?)
data_item_id,block_height,raw_data_item,status,last_error,first_seen_at_ms,updated_at_ms
) VALUES(?,?,?,?, '',?,?)
ON CONFLICT(data_item_id) DO NOTHING
""";
try (Connection c = db.getConnection(); PreparedStatement ps = c.prepareStatement(sql)) {
ps.setBytes(1, dataItemId);
ps.setString(2, rootTxId);
ps.setLong(3, blockHeight);
ps.setBytes(4, rawDataItem);
ps.setString(5, STATUS_PENDING);
ps.setLong(2, blockHeight);
ps.setBytes(3, rawDataItem);
ps.setString(4, STATUS_PENDING);
ps.setLong(5, nowMs);
ps.setLong(6, nowMs);
ps.setLong(7, nowMs);
return ps.executeUpdate() > 0;
}
}
@@ -82,7 +81,7 @@ public final class ArweaveBlockImportDAO {
public List<QueueItem> listPending(int limit) throws SQLException {
int safeLimit = Math.max(1, Math.min(limit, 100_000));
String sql = """
SELECT data_item_id,root_tx_id,block_height,raw_data_item,status,last_error,first_seen_at_ms,updated_at_ms
SELECT data_item_id,block_height,raw_data_item,status,last_error,first_seen_at_ms,updated_at_ms
FROM arweave_block_import_queue
WHERE status=?
ORDER BY block_height ASC, first_seen_at_ms ASC
@@ -95,7 +94,7 @@ public final class ArweaveBlockImportDAO {
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
out.add(new QueueItem(
rs.getBytes("data_item_id"), rs.getString("root_tx_id"), rs.getLong("block_height"),
rs.getBytes("data_item_id"), rs.getLong("block_height"),
rs.getBytes("raw_data_item"), rs.getString("status"), rs.getString("last_error"),
rs.getLong("first_seen_at_ms"), rs.getLong("updated_at_ms")));
}
@@ -27,9 +27,9 @@ public final class BlocksDAO {
login,bch_name,block_number,msg_type,msg_sub_type,block_bytes,
to_login,to_bch_name,to_block_number,to_block_hash,
block_hash,block_signature,data_item_id,
arweave_publish_pending,arweave_published_at_ms,arweave_root_tx_id,
arweave_publish_pending,arweave_published_at_ms,
edited_by_block_number,line_code,prev_line_number,prev_line_hash,this_line_number
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""";
try (PreparedStatement ps = c.prepareStatement(sql)) {
int i = 1;
@@ -48,7 +48,6 @@ public final class BlocksDAO {
ps.setBytes(i++, e.getDataItemId());
ps.setBoolean(i++, e.isArweavePublishPending());
if (e.getArweavePublishedAtMs() == null) ps.setNull(i++, Types.BIGINT); else ps.setLong(i++, e.getArweavePublishedAtMs());
setNullableString(ps, i++, e.getArweaveRootTxId());
setNullableInt(ps, i++, e.getEditedByBlockNumber());
setNullableInt(ps, i++, e.getLineCode());
setNullableInt(ps, i++, e.getPrevLineNumber());
@@ -74,7 +73,7 @@ public final class BlocksDAO {
String sql = """
SELECT login,bch_name,block_number,msg_type,msg_sub_type,block_bytes,
to_login,to_bch_name,to_block_number,to_block_hash,block_hash,block_signature,data_item_id,
arweave_publish_pending,arweave_published_at_ms,arweave_root_tx_id,
arweave_publish_pending,arweave_published_at_ms,
edited_by_block_number,line_code,prev_line_number,prev_line_hash,this_line_number
FROM blocks
WHERE arweave_publish_pending = TRUE
@@ -89,18 +88,18 @@ public final class BlocksDAO {
return out;
}
public void markArweavePublished(List<byte[]> dataItemIds, String rootTxId, long publishedAtMs) throws SQLException {
public void markArweavePublished(List<byte[]> dataItemIds, long publishedAtMs) throws SQLException {
if (dataItemIds == null || dataItemIds.isEmpty()) return;
String sql = """
UPDATE blocks
SET arweave_publish_pending=FALSE, arweave_published_at_ms=?, arweave_root_tx_id=?
SET arweave_publish_pending=FALSE, arweave_published_at_ms=?
WHERE data_item_id=?
""";
try (Connection c = db.getConnection()) {
c.setAutoCommit(false);
try (PreparedStatement ps = c.prepareStatement(sql)) {
for (byte[] id : dataItemIds) {
ps.setLong(1, publishedAtMs); ps.setString(2, rootTxId); ps.setBytes(3, id); ps.addBatch();
ps.setLong(1, publishedAtMs); ps.setBytes(2, id); ps.addBatch();
}
ps.executeBatch(); c.commit();
} catch (Exception e) { c.rollback(); throw e; }
@@ -133,7 +132,7 @@ public final class BlocksDAO {
private static String baseSelect(){return """
SELECT login,bch_name,block_number,msg_type,msg_sub_type,block_bytes,
to_login,to_bch_name,to_block_number,to_block_hash,block_hash,block_signature,data_item_id,
arweave_publish_pending,arweave_published_at_ms,arweave_root_tx_id,
arweave_publish_pending,arweave_published_at_ms,
edited_by_block_number,line_code,prev_line_number,prev_line_hash,this_line_number
FROM blocks
""";}
@@ -144,7 +143,7 @@ public final class BlocksDAO {
e.setMsgType(rs.getInt("msg_type")); e.setMsgSubType(rs.getInt("msg_sub_type")); e.setBlockBytes(rs.getBytes("block_bytes"));
e.setToLogin(rs.getString("to_login")); e.setToBchName(rs.getString("to_bch_name")); e.setToBlockNumber((Integer)rs.getObject("to_block_number")); e.setToBlockHash(rs.getBytes("to_block_hash"));
e.setBlockHash(rs.getBytes("block_hash")); e.setBlockSignature(rs.getBytes("block_signature")); e.setDataItemId(rs.getBytes("data_item_id"));
e.setArweavePublishPending(rs.getBoolean("arweave_publish_pending")); e.setArweavePublishedAtMs((Long)rs.getObject("arweave_published_at_ms")); e.setArweaveRootTxId(rs.getString("arweave_root_tx_id"));
e.setArweavePublishPending(rs.getBoolean("arweave_publish_pending")); e.setArweavePublishedAtMs((Long)rs.getObject("arweave_published_at_ms"));
e.setEditedByBlockNumber((Integer)rs.getObject("edited_by_block_number")); e.setLineCode((Integer)rs.getObject("line_code")); e.setPrevLineNumber((Integer)rs.getObject("prev_line_number")); e.setPrevLineHash(rs.getBytes("prev_line_hash")); e.setThisLineNumber((Integer)rs.getObject("this_line_number"));
return e;
}
@@ -40,7 +40,6 @@ public class BlockEntry {
private byte[] dataItemId;
private boolean arweavePublishPending;
private Long arweavePublishedAtMs;
private String arweaveRootTxId;
private Integer editedByBlockNumber;
@@ -95,8 +94,6 @@ public class BlockEntry {
public void setArweavePublishPending(boolean arweavePublishPending) { this.arweavePublishPending = arweavePublishPending; }
public Long getArweavePublishedAtMs() { return arweavePublishedAtMs; }
public void setArweavePublishedAtMs(Long arweavePublishedAtMs) { this.arweavePublishedAtMs = arweavePublishedAtMs; }
public String getArweaveRootTxId() { return arweaveRootTxId; }
public void setArweaveRootTxId(String arweaveRootTxId) { this.arweaveRootTxId = arweaveRootTxId; }
public Integer getEditedByBlockNumber() { return editedByBlockNumber; }
public void setEditedByBlockNumber(Integer editedByBlockNumber) { this.editedByBlockNumber = editedByBlockNumber; }
@@ -0,0 +1,11 @@
-- Turbo/direct-Arweave transport no longer stores root transaction IDs for user DataItems.
ALTER TABLE blocks
DROP COLUMN IF EXISTS arweave_root_tx_id;
ALTER TABLE arweave_block_import_queue
DROP COLUMN IF EXISTS root_tx_id;
UPDATE db_schema_version
SET schema_version=25,
updated_at_ms=CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT)
WHERE id=1;
@@ -500,7 +500,6 @@ CREATE TABLE IF NOT EXISTS blocks (
data_item_id BYTEA NOT NULL,
arweave_publish_pending BOOLEAN NOT NULL DEFAULT FALSE,
arweave_published_at_ms BIGINT,
arweave_root_tx_id TEXT,
edited_by_block_number INTEGER CHECK (edited_by_block_number IS NULL OR edited_by_block_number >= 0),
line_code INTEGER CHECK (line_code IS NULL OR line_code >= 0),
prev_line_number INTEGER CHECK (prev_line_number IS NULL OR prev_line_number >= 0),
@@ -535,7 +534,6 @@ VALUES (1, 0, 0) ON CONFLICT (id) DO NOTHING;
CREATE TABLE IF NOT EXISTS arweave_block_import_queue (
data_item_id BYTEA PRIMARY KEY,
root_tx_id TEXT NOT NULL,
block_height BIGINT NOT NULL,
raw_data_item BYTEA NOT NULL,
status TEXT NOT NULL,
@@ -2081,7 +2079,7 @@ CREATE INDEX IF NOT EXISTS idx_solana_user_pda_current_archive_pending
WHERE archive_head_tx_id <> '';
INSERT INTO db_schema_version(id,schema_version,updated_at_ms)
VALUES(1,24,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
VALUES(1,25,CAST(EXTRACT(EPOCH FROM clock_timestamp())*1000 AS BIGINT))
ON CONFLICT(id) DO UPDATE SET schema_version=EXCLUDED.schema_version, updated_at_ms=EXCLUDED.updated_at_ms;
COMMIT;
@@ -409,10 +409,10 @@ public final class Net_AddBlock_Handler implements JsonMessageHandler {
channelMetaUpdateEntry.setMetaUpdatedAtMs(block.timestamp * 1000L);
}
// Channel DataItems are indexed by a signed canonical channel slug tag: c=<slug>.
// Channel DataItems are indexed by a signed canonical channel slug tag: c_test5590=<slug>.
try {
String expectedChannelSlug = expectedChannelSlug(blockchainName, block, channelNameStateEntry);
String actualChannelSlug = block.getDataItem().tagValue("c");
String actualChannelSlug = block.getDataItem().tagValue("c_test5590");
if (expectedChannelSlug != null) {
if (!expectedChannelSlug.equals(actualChannelSlug)) {
return new AddBlockResult(WireCodes.Status.BAD_REQUEST, "bad_channel_tag", serverLastNum, serverLastHashHex);
@@ -138,11 +138,14 @@ test.freeAvatar.walletJwkPath=
# ============================================================
# Arweave per-user-block transport (ANS-104)
# Test namespace: each user DataItem is signed with App=test5590.
# Channel DataItems additionally contain c=<canonical-channel-slug>.
# Channel DataItems additionally contain c_test5590=<canonical-channel-slug>.
# publish.mode: turbo | arweave | none
# ============================================================
arweave.blocks.publish.enabled=false
arweave.blocks.publish.mode=none
arweave.blocks.publish.intervalMinutes=15
arweave.blocks.publish.maxItems=10000
# Direct Arweave L1 fallback: server combines user DataItems into one standard ANS-104 bundle.
arweave.blocks.publish.maxBundleBytes=134217728
arweave.blocks.publish.gateway=https://arweave.net
arweave.blocks.publish.walletJwkPath=
@@ -150,10 +153,18 @@ arweave.blocks.publish.minConfirmations=0
arweave.blocks.publish.confirmPollSeconds=30
arweave.blocks.publish.confirmTimeoutMinutes=180
# Turbo: uploads each already user-signed DataItem separately, without re-signing it.
# paidByAddress is the public Turbo payer address. If it is empty and turbo.walletJwkPath is set,
# the Arweave payer address is derived locally from that JWK. The private key is never sent to Turbo.
# For paid uploads of someone else's signed DataItem, Turbo Credit Share Approval must exist for its signer.
arweave.blocks.publish.turbo.uploadUrl=https://turbo.ardrive.io/tx
arweave.blocks.publish.turbo.paidByAddress=
arweave.blocks.publish.turbo.walletJwkPath=
arweave.blocks.sync.enabled=false
arweave.blocks.sync.intervalMinutes=15
arweave.blocks.sync.gateway=https://turbo-gateway.com
arweave.blocks.sync.pageSize=100
arweave.blocks.sync.queueBatchSize=10000
arweave.blocks.sync.startBlockHeight=0
arweave.blocks.sync.maxRootBundleBytes=268435456
arweave.blocks.sync.maxDataItemBytes=8388608
@@ -82,7 +82,7 @@ public final class AddBlockSender {
List<Ans104DataItem.Tag> tags = new ArrayList<>();
tags.add(new Ans104DataItem.Tag("App", "test5590"));
String channelSlug = channelSlugFor(body);
if (channelSlug != null) tags.add(new Ans104DataItem.Tag("c", channelSlug));
if (channelSlug != null) tags.add(new Ans104DataItem.Tag("c_test5590", channelSlug));
byte[] signingMessage = Ans104DataItem.buildSigningMessage(owner32, tags, frame);
byte[] signature64 = utils.crypto.Ed25519Util.sign(signingMessage, loginPrivKey);