commit c8033ebd84c75a950f7352c4594f5efbdce42e33
parent 45c56db5ce7b80378d4bd8b9768c609ba601ef23
Author: triesap <triesap@radroots.dev>
Date: Sun, 28 Dec 2025 19:56:32 +0000
tangle: add Nostr sync + tar.gz export pipeline
- Add tangle-events-wasm dependency and init in wasm bootstrap
- Expose nostr_sync_all API with bundle parsing, signing, and relay publish
- Switch database export format from .zip to .tar.gz with tar writer + gzip
- Harden export/share flows with user activation and permission error handling
Diffstat:
3 files changed, 375 insertions(+), 34 deletions(-)
diff --git a/client/package.json b/client/package.json
@@ -90,6 +90,7 @@
"@radroots/http": "workspace:*",
"@radroots/tangle-db-schema-bindings": "workspace:*",
"@radroots/tangle-db-wasm": "workspace:*",
+ "@radroots/tangle-events-wasm": "workspace:*",
"@radroots/types-bindings": "workspace:*",
"@radroots/utils": "workspace:*",
"@radroots/nostr": "workspace:*",
diff --git a/client/src/tangle/types.ts b/client/src/tangle/types.ts
@@ -160,7 +160,9 @@ import { type SqlJsMigrationState } from "../sql/types.js";
import type { IError } from "@radroots/types-bindings";
import type {
TangleDatabaseExportOptions,
- TangleDatabaseJsonExport
+ TangleDatabaseJsonExport,
+ TangleNostrSyncOptions,
+ TangleNostrSyncSummary
} from "./web.js";
export interface IClientTangleDatabase {
@@ -173,6 +175,7 @@ export interface IClientTangleDatabase {
export_json(): Promise<TangleDatabaseJsonExport | IError<string>>;
import_json(backup: TangleDatabaseJsonExport): Promise<void | IError<string>>;
export_database(opts: TangleDatabaseExportOptions): Promise<void | IError<string>>;
+ nostr_sync_all(opts: TangleNostrSyncOptions): Promise<TangleNostrSyncSummary | IError<string>>;
farm_create(opts: IFarmCreate): Promise<IFarmCreateResolve | IError<string>>;
farm_find_one(opts: IFarmFindOne): Promise<IFarmFindOneResolve | IError<string>>;
farm_find_many(opts?: IFarmFindMany): Promise<IFarmFindManyResolve | IError<string>>;
diff --git a/client/src/tangle/web.ts b/client/src/tangle/web.ts
@@ -246,6 +246,19 @@ import init_wasm, {
tangle_db_export_json,
tangle_db_import_json
} from "@radroots/tangle-db-wasm";
+import init_tangle_events_wasm, {
+ tangle_events_ingest_event,
+ tangle_events_sync_all
+} from "@radroots/tangle-events-wasm";
+import {
+ nostr_context_create,
+ nostr_event_sign,
+ nostr_public_key_from_secret,
+ nostr_publish,
+ nostr_relays_clear,
+ nostr_relays_open,
+ type NostrContext
+} from "@radroots/nostr";
import type { IError } from "@radroots/types-bindings";
import { err_msg, handle_err, type IdbClientConfig } from "@radroots/utils";
import { IDB_CONFIG_TANGLE } from "../idb/config.js";
@@ -313,8 +326,8 @@ export type TangleDatabaseExportManifestTs = {
};
export type TangleDatabaseExportManifest = {
- rs: TangleDatabaseExportManifestRs;
- ts: TangleDatabaseExportManifestTs;
+ rust: TangleDatabaseExportManifestRs;
+ client: TangleDatabaseExportManifestTs;
};
export type TangleDatabaseExportSnapshot = {
@@ -336,6 +349,37 @@ export type TangleDatabaseExportOptions = {
signer?: TangleDatabaseExportSigner;
};
+export type TangleNostrSyncSigner = {
+ secret_key: string;
+};
+
+export type TangleNostrEventDraft = {
+ kind: number;
+ author: string;
+ content: string;
+ tags: string[][];
+};
+
+export type TangleNostrSyncBundle = {
+ version: number;
+ events: TangleNostrEventDraft[];
+};
+
+export type TangleNostrSyncOptions = {
+ relays: string[];
+ signers: TangleNostrSyncSigner[];
+ publish_timeout_ms?: number;
+ context?: NostrContext;
+};
+
+export type TangleNostrSyncSummary = {
+ events_total: number;
+ events_published: number;
+ events_failed: number;
+ events_skipped: number;
+ missing_signers: string[];
+};
+
export type WebTangleDatabaseConfig = {
store_key?: string;
idb_config?: IdbClientConfig;
@@ -417,6 +461,67 @@ const is_export_snapshot = (value: unknown): value is TangleDatabaseExportSnapsh
return true;
};
+const is_string_list = (value: unknown): value is string[] =>
+ Array.isArray(value) && value.every((item) => typeof item === "string");
+
+const is_tag_list = (value: unknown): value is string[][] =>
+ Array.isArray(value) && value.every(is_string_list);
+
+const is_tangle_nostr_event_draft = (value: unknown): value is TangleNostrEventDraft => {
+ if (!is_record(value)) return false;
+ if (typeof value.kind !== "number" || !Number.isFinite(value.kind)) return false;
+ if (typeof value.author !== "string") return false;
+ if (typeof value.content !== "string") return false;
+ if (!is_tag_list(value.tags)) return false;
+ return true;
+};
+
+const is_tangle_nostr_sync_bundle = (value: unknown): value is TangleNostrSyncBundle => {
+ if (!is_record(value)) return false;
+ if (typeof value.version !== "number" || !Number.isFinite(value.version)) return false;
+ if (!Array.isArray(value.events) || !value.events.every(is_tangle_nostr_event_draft)) return false;
+ return true;
+};
+
+const parse_tangle_nostr_sync_bundle = (value: unknown): TangleNostrSyncBundle | IError<string> => {
+ let parsed: unknown = value;
+ if (typeof value === "string") {
+ try {
+ parsed = JSON.parse(value);
+ } catch {
+ return err_msg(cl_tangle_error.parse_failure);
+ }
+ }
+ if (!is_tangle_nostr_sync_bundle(parsed)) return err_msg(cl_tangle_error.invalid_response);
+ return parsed;
+};
+
+const is_ingest_outcome = (value: unknown): value is "applied" | "skipped" =>
+ value === "applied" || value === "skipped";
+
+const tangle_sync_event_d_tag = (tags: string[][]): string => {
+ const match = tags.find((tag) => tag[0] === "d");
+ const value = match?.[1];
+ return typeof value === "string" ? value : "";
+};
+
+const tangle_sync_event_key = (draft: TangleNostrEventDraft): string =>
+ `${draft.kind}:${draft.author}:${tangle_sync_event_d_tag(draft.tags)}`;
+
+const build_signer_map = (signers: TangleNostrSyncSigner[]): Record<string, string> => {
+ const map: Record<string, string> = {};
+ for (const signer of signers) {
+ const secret_key = signer.secret_key;
+ if (!secret_key || typeof secret_key !== "string") continue;
+ const pubkey = nostr_public_key_from_secret(secret_key);
+ map[pubkey] = secret_key;
+ }
+ return map;
+};
+
+const publish_results_has_success = (results: Record<string, { status?: string }>): boolean =>
+ Object.values(results).some((result) => result.status === "success");
+
type ZipEntry = {
name: string;
data: Uint8Array;
@@ -612,6 +717,107 @@ const zip_write_stream = async (stream: ZipFileWritable, entries: ZipEntry[]): P
await stream.close();
};
+type TarEntry = {
+ name: string;
+ data: Uint8Array;
+};
+
+const TAR_BLOCK_SIZE = 512;
+
+const tar_pad_size = (size: number): number => {
+ const rem = size % TAR_BLOCK_SIZE;
+ return rem === 0 ? 0 : TAR_BLOCK_SIZE - rem;
+};
+
+const tar_write_string = (buf: Uint8Array, offset: number, length: number, value: string): void => {
+ const enc = new TextEncoder();
+ const bytes = enc.encode(value);
+ const slice = bytes.length > length ? bytes.slice(0, length) : bytes;
+ buf.set(slice, offset);
+};
+
+const tar_write_octal = (buf: Uint8Array, offset: number, length: number, value: number): void => {
+ const str = value.toString(8).padStart(length - 1, "0");
+ tar_write_string(buf, offset, length - 1, str);
+ buf[offset + length - 1] = 0;
+};
+
+const tar_header = (entry: TarEntry, mtime: number): Uint8Array => {
+ if (entry.name.length > 100) throw new Error("tar entry name too long");
+ const header = new Uint8Array(TAR_BLOCK_SIZE);
+ tar_write_string(header, 0, 100, entry.name);
+ tar_write_octal(header, 100, 8, 0o644);
+ tar_write_octal(header, 108, 8, 0);
+ tar_write_octal(header, 116, 8, 0);
+ tar_write_octal(header, 124, 12, entry.data.length);
+ tar_write_octal(header, 136, 12, mtime);
+ for (let i = 148; i < 156; i++) header[i] = 32;
+ header[156] = 48;
+ tar_write_string(header, 257, 6, "ustar");
+ tar_write_string(header, 263, 2, "00");
+ let checksum = 0;
+ for (let i = 0; i < header.length; i++) checksum += header[i];
+ const chk = checksum.toString(8).padStart(6, "0");
+ tar_write_string(header, 148, 6, chk);
+ header[154] = 0;
+ header[155] = 32;
+ return header;
+};
+
+const tar_build_bytes = (entries: TarEntry[], mtime: number): Uint8Array => {
+ const parts: Uint8Array[] = [];
+ let total = 0;
+ for (const entry of entries) {
+ const header = tar_header(entry, mtime);
+ const pad = tar_pad_size(entry.data.length);
+ parts.push(header, entry.data);
+ total += header.length + entry.data.length;
+ if (pad) {
+ const padding = new Uint8Array(pad);
+ parts.push(padding);
+ total += padding.length;
+ }
+ }
+ const end = new Uint8Array(TAR_BLOCK_SIZE * 2);
+ parts.push(end);
+ total += end.length;
+ const out = new Uint8Array(total);
+ let offset = 0;
+ for (const part of parts) {
+ out.set(part, offset);
+ offset += part.length;
+ }
+ return out;
+};
+
+const tar_stream = (entries: TarEntry[], mtime: number): ReadableStream<Uint8Array> => {
+ return new ReadableStream({
+ start(controller) {
+ for (const entry of entries) {
+ const header = tar_header(entry, mtime);
+ controller.enqueue(header);
+ controller.enqueue(entry.data);
+ const pad = tar_pad_size(entry.data.length);
+ if (pad) controller.enqueue(new Uint8Array(pad));
+ }
+ controller.enqueue(new Uint8Array(TAR_BLOCK_SIZE * 2));
+ controller.close();
+ }
+ });
+};
+
+const gzip_bytes = async (bytes: Uint8Array): Promise<Uint8Array> => {
+ if (typeof CompressionStream === "undefined") {
+ throw new Error("tangle export requires gzip support");
+ }
+ const stream = new CompressionStream("gzip");
+ const writer = stream.writable.getWriter();
+ writer.write(bytes);
+ await writer.close();
+ const buffer = await new Response(stream.readable).arrayBuffer();
+ return new Uint8Array(buffer);
+};
+
const bytes_to_hex = (bytes: Uint8Array): string => {
const hex: string[] = [];
for (let i = 0; i < bytes.length; i++) {
@@ -635,10 +841,9 @@ const filename_slug = (value: string): string => {
return `radroots-${slug}`;
};
-const export_filename = (app_name: string, app_version: string, exported_at: string): string => {
- const ts = exported_at.replace(/[:.]/g, "-");
+const export_filename = (app_name: string, app_version: string): string => {
const base = filename_slug(app_name);
- return `${base}-${app_version}-backup-${ts}.zip`;
+ return `${base}-${app_version}-backup.tar.gz`;
};
const get_zip_file_picker = (): ZipFilePicker | undefined => {
@@ -647,6 +852,26 @@ const get_zip_file_picker = (): ZipFilePicker | undefined => {
return picker;
};
+const user_activation_is_active = (): boolean => {
+ if (typeof navigator === "undefined") return false;
+ const nav = navigator as Navigator & { userActivation?: { isActive?: boolean } };
+ if (!nav.userActivation) return true;
+ return nav.userActivation.isActive === true;
+};
+
+const is_permission_error = (err: unknown): boolean => {
+ if (!err) return false;
+ if (typeof err === "string") {
+ const msg = err.toLowerCase();
+ return msg.includes("permission") || msg.includes("denied") || msg.includes("not allowed");
+ }
+ if (!is_record(err)) return false;
+ const name = typeof err.name === "string" ? err.name.toLowerCase() : "";
+ const message = typeof err.message === "string" ? err.message.toLowerCase() : "";
+ if (name.includes("notallowed") || name.includes("abort")) return true;
+ return message.includes("permission") || message.includes("denied") || message.includes("not allowed");
+};
+
const can_share_file = (file: File): boolean => {
if (typeof navigator === "undefined") return false;
const nav = navigator as Navigator & { canShare?: (data: { files?: File[] }) => boolean };
@@ -672,28 +897,39 @@ const download_blob = (blob: Blob, filename: string): void => {
URL.revokeObjectURL(url);
};
-const export_zip = async (filename: string, entries: ZipEntry[]): Promise<void> => {
+const export_tar_gz = async (filename: string, entries: TarEntry[], mtime: number): Promise<void> => {
const picker = get_zip_file_picker();
- if (picker) {
- const handle = await picker({
- suggestedName: filename,
- types: [
- {
- description: "Radroots tangle export",
- accept: { "application/zip": [".zip"] }
- }
- ]
- });
- const stream = await handle.createWritable();
- await zip_write_stream(stream, entries);
- return;
- }
- const zip_bytes = zip_build_bytes(entries);
- const blob = new Blob([zip_bytes], { type: "application/zip" });
- const file = new File([blob], filename, { type: "application/zip" });
- if (can_share_file(file)) {
- const shared = await share_file(file);
- if (shared) return;
+ if (picker && user_activation_is_active() && typeof CompressionStream !== "undefined") {
+ try {
+ const handle = await picker({
+ suggestedName: filename,
+ types: [
+ {
+ description: "Radroots tangle export",
+ accept: { "application/gzip": [".tar.gz"] }
+ }
+ ]
+ });
+ const stream = await handle.createWritable();
+ await tar_stream(entries, mtime)
+ .pipeThrough(new CompressionStream("gzip"))
+ .pipeTo(stream);
+ return;
+ } catch (e) {
+ if (!is_permission_error(e)) throw e;
+ }
+ }
+ const tar_bytes = tar_build_bytes(entries, mtime);
+ const gz_bytes = await gzip_bytes(tar_bytes);
+ const blob = new Blob([gz_bytes], { type: "application/gzip" });
+ const file = new File([blob], filename, { type: "application/gzip" });
+ if (can_share_file(file) && user_activation_is_active()) {
+ try {
+ const shared = await share_file(file);
+ if (shared) return;
+ } catch (e) {
+ if (!is_permission_error(e)) throw e;
+ }
}
download_blob(blob, filename);
};
@@ -710,6 +946,7 @@ const wasm_init_once = async (): Promise<void> => {
if (!wasm_init_promise) {
wasm_init_promise = (async () => {
await init_wasm();
+ await init_tangle_events_wasm();
})();
}
try {
@@ -887,6 +1124,105 @@ export class WebTangleDatabase implements IWebTangleDatabase {
}
}
+ async nostr_sync_all(opts: TangleNostrSyncOptions): Promise<TangleNostrSyncSummary | IError<string>> {
+ try {
+ await this.ensure_ready();
+ const relays = Array.from(new Set(opts.relays.map((relay) => relay.trim()).filter((relay) => relay.length)));
+ if (!relays.length) return err_msg(`tangle sync requires relays`);
+ if (!opts.signers.length) return err_msg(`tangle sync requires signers`);
+ const signer_map = build_signer_map(opts.signers);
+ if (!Object.keys(signer_map).length) return err_msg(`tangle sync requires valid signers`);
+
+ const farms = await this.farm_find_many();
+ if ("err" in farms) return farms;
+ const event_map: Record<string, TangleNostrEventDraft> = {};
+ for (const farm of farms.results) {
+ const bundle_raw = tangle_events_sync_all(this.serialize({
+ farm: { id: farm.id },
+ options: null
+ }));
+ const bundle = parse_tangle_nostr_sync_bundle(bundle_raw);
+ if ("err" in bundle) return bundle;
+ for (const draft of bundle.events) {
+ const key = tangle_sync_event_key(draft);
+ if (!event_map[key]) event_map[key] = draft;
+ }
+ }
+
+ const event_keys = Object.keys(event_map);
+ event_keys.sort();
+ if (!event_keys.length) {
+ return {
+ events_total: 0,
+ events_published: 0,
+ events_failed: 0,
+ events_skipped: 0,
+ missing_signers: []
+ };
+ }
+
+ const context = opts.context ?? nostr_context_create();
+ const context_owned = !opts.context;
+ let events_published = 0;
+ let events_failed = 0;
+ let events_skipped = 0;
+ const missing_signers = new Set<string>();
+
+ try {
+ nostr_relays_open(context, relays);
+ for (const key of event_keys) {
+ const draft = event_map[key];
+ const secret_key = signer_map[draft.author];
+ if (!secret_key) {
+ missing_signers.add(draft.author);
+ events_skipped += 1;
+ continue;
+ }
+ const event = nostr_event_sign({
+ secret_key,
+ event: {
+ kind: draft.kind,
+ created_at: Math.floor(Date.now() / 1000),
+ tags: draft.tags,
+ content: draft.content
+ }
+ });
+ const publish_results = await nostr_publish({
+ event,
+ relays,
+ context,
+ timeout: opts.publish_timeout_ms
+ });
+ if (!publish_results_has_success(publish_results)) {
+ events_failed += 1;
+ continue;
+ }
+ const ingest_result = tangle_events_ingest_event(this.serialize(event));
+ if (!is_ingest_outcome(ingest_result)) {
+ events_failed += 1;
+ continue;
+ }
+ events_published += 1;
+ }
+ } finally {
+ if (context_owned) nostr_relays_clear(context);
+ }
+
+ const summary: TangleNostrSyncSummary = {
+ events_total: event_keys.length,
+ events_published,
+ events_failed,
+ events_skipped,
+ missing_signers: Array.from(missing_signers)
+ };
+ if (summary.missing_signers.length) return err_msg(`tangle sync missing signers: ${summary.missing_signers.join(", ")}`);
+ if (summary.events_failed) return err_msg(`tangle sync publish failed (${summary.events_failed}/${summary.events_total})`);
+ return summary;
+ } catch (e) {
+ return handle_err(e);
+ }
+ }
+
private async export_database_inner(opts: TangleDatabaseExportOptions): Promise<void> {
await this.ensure_ready();
const app_name = opts.app_name;
@@ -913,16 +1249,16 @@ export class WebTangleDatabase implements IWebTangleDatabase {
};
let manifest: TangleDatabaseExportManifest = {
- rs: manifest_rs,
- ts: manifest_ts_base
+ rust: manifest_rs,
+ client: manifest_ts_base
};
if (opts.signer) {
const nostr_event = await opts.signer({ db_sha256, manifest });
if (nostr_event) {
manifest = {
- rs: manifest_rs,
- ts: {
+ rust: manifest_rs,
+ client: {
...manifest_ts_base,
nostr_event
}
@@ -932,11 +1268,12 @@ export class WebTangleDatabase implements IWebTangleDatabase {
const manifest_json = JSON.stringify(manifest, null, 2);
const manifest_bytes = new TextEncoder().encode(manifest_json);
- const filename = export_filename(app_name, app_version, exported_at);
- await export_zip(filename, [
+ const filename = export_filename(app_name, app_version);
+ const mtime = Math.floor(Date.parse(exported_at) / 1000);
+ await export_tar_gz(filename, [
{ name: "manifest.json", data: manifest_bytes },
{ name: "tangle.db", data: db_bytes }
- ]);
+ ], mtime);
} finally {
if (export_active) tangle_db_export_finish();
}
