Files
hyperframes/packages/share/src/reviewExport.mjs
T

943 lines
41 KiB
JavaScript

// @ts-check
// Adding a version to a project's review page: the newest MP4 export goes up, then the server
// processes and screens it. Main owns the run, so closing the Share popover never stops it.
import { ShareBusyError, withProjectShareLock } from "./projectLock.mjs";
import { runFile } from "./host.mjs";
import { randomUUID } from "node:crypto";
import { stat } from "node:fs/promises";
import path from "node:path";
import { ffBinary, latestExportFile, normalize, projectFingerprint } from "./host.mjs";
import { mkdirInside } from "./storage.mjs";
import {
isJsonObject,
isNullableSavedId,
isOptionalBoolean,
isOptionalSavedId,
isSavedId,
readJsonRecord,
SharingStateError,
writeJsonAtomic,
} from "./storage.mjs";
import { ReviewFailed } from "./reviewClient.mjs";
import { readFromReview, sharedFromReview } from "./fromReview.mjs";
import { reviewProjectId } from "./reviewShare.mjs";
import { fileSha256Base64, putFile } from "./reviewUpload.mjs";
import { versionSentence } from "./reviewVersionSentences.mjs";
/** @typedef {import("./reviewClient.mjs").ReviewClient} ReviewClient
* @typedef {import("./types/reviewWire.js").ExportDTO} ExportDTO @typedef {import("./types/reviewWire.js").ExportRequest} ExportRequest
* @typedef {import("./types/share.js").ReviewVersion} ReviewVersion */
/** @typedef {import("./types/share.js").ReviewExportProgress} ReviewExportProgress
* @typedef {{ duration: number, width: number, height: number }} VideoFacts
* @typedef {import("./fromReview.mjs").FromReview} FromReview
* @typedef {{ since: number, value?: "yes" | "no" }} ReviewSent
* @typedef {{ files: boolean, first: boolean, fromReview?: "yes" | "no" }} Added */
/** `pending`: the create body as sent and its video once answered, so a retry is the same; `snapshotRefused`: the
* server refused its snapshot, so it goes up video-only; `unanswered`: its create may have gone out without a
* definitive result; `reviewSent`: cleanup cutoff, with attribution only when the accepted attempt is
* known; `last`: the newest version.
* @typedef {{ pending: { body: ExportRequest, videoId: string | null, projectId?: string | null,
* renderKey?: string, checkingSince?: number, snapshotRefused?: boolean, unanswered?: boolean, reviewSent?: ReviewSent } | null,
* last: { fingerprint: string, versionId: string, exportedAt: string, projectId: string, files?: boolean,
* snapshot?: ExportRequest["snapshot"], checksum?: string }
* | null, failure?: (Failure & { projectId: string }) | null }} VersionRecord */
/** `onAdded`: a version went ready, with whether it carries files and whether it is the page's first from here.
* @typedef {{ onProgress?: (progress: ReviewExportProgress | null) => void, onUploaded?: () => Promise<void>,
* open?: () => boolean, onAdded?: (added: Added) => void }} VersionHooks */
/** `projectId` overrides the recorded review project; `open`: a held video is watched only while the project is open.
* `filesForExport`: the synced snapshot a new version carries, null for
* none. `allowPending` lets a host treat accepted processing or screening as a successful return.
* @typedef {VersionHooks & { projectPath: string, title: string, client: ReviewClient, put?: typeof putFile,
* sleep?: (ms: number) => Promise<void>, probe?: (file: string) => Promise<VideoFacts | null>, now?: () => number,
* projectId?: string | null, filesForExport?: FilesForRender, fromReview?: FromReview | null,
* allowPending?: boolean, requireSource?: boolean, omitSource?: boolean }} VersionRequest */
/** The snapshot of `rendered`, the project as the export was rendered; null when the folder is not that render.
* @typedef {(projectPath: string, rendered: string | null) => Promise<import("./types/reviewWire.js").SnapshotRef | null>}
* FilesForRender */
const POLL_MS = 2_000;
/** Screening past this says it can take a while and is polled every SLOW_POLL_MS: staff review has no time limit. */
const SLOW_AFTER_MS = 2 * 60_000;
/** A server that says how screening stands: checking past this since the upload says it is taking longer than usual,
* and the watch keeps going as long again after it. */
const LATE_AFTER_MS = 15 * 60_000;
const SLOW_POLL_MS = 30_000;
/** Refusals the same video would get again: no Try again. */
const FINAL = new Set(["export_blocked", "content_rejected", "feature_unavailable"]);
/** Too big or too long: the same file is refused again, but a smaller or shorter export can go up. */
const TOO_BIG = new Set(["export_too_large", "export_too_long"]);
/** Longer than the backend's own limiter ever asks, and short enough that a laptop waking up checks again soon. */
const MAX_WAIT_MS = 5 * 60_000;
/** A run stops watching after this long; the video stays recorded, so the next press reads it back. */
const WATCH_MS = 15 * 60_000;
/** A held video waits on a person at HeyGen, which the sentence says can take up to a day: it is watched that long,
* at SLOW_POLL_MS, while its project stays open. */
const HELD_WATCH_MS = 24 * 60 * 60_000;
/** The backend's limits, checked before hashing or uploading anything. */
const MAX_BYTES = 2 * 1024 ** 3;
const MAX_SECONDS = 30 * 60;
const run$ = runFile;
const statePath = (/** @type {string} */ dir) =>
path.join(dir, ".hyperframes", "review-export.json");
/** The MP4's own picture size and length: the backend checks the upload against them, so never the canvas.
* @param {string} file @returns {Promise<VideoFacts | null>} */
async function probeVideo(file) {
try {
const args = [
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height:format=duration",
"-of",
"json",
"--",
file,
];
const { stdout } = await run$(ffBinary("ffprobe"), args);
return factsOf(stdout);
} catch {
return null;
}
}
/** ffprobe's JSON as facts, or null when it names no picture or no length. @param {string} stdout */
export function factsOf(stdout) {
const facts = JSON.parse(stdout);
const { width, height } = facts.streams?.[0] ?? {};
const duration = Number.parseFloat(facts.format?.duration);
return width > 0 && height > 0 && duration > 0 ? { duration, width, height } : null;
}
/** @param {unknown} upload */
const isSavedUpload = (upload) =>
isJsonObject(upload) &&
Number.isSafeInteger(upload.size_bytes) &&
isSavedId(upload.checksum_sha256) &&
upload.content_type === "video/mp4";
/** @param {unknown} snapshot */
const isOptionalSnapshot = (snapshot) =>
snapshot === undefined ||
(isJsonObject(snapshot) &&
isSavedId(snapshot.space_id) &&
isSavedId(snapshot.composition_id) &&
isSavedId(snapshot.snapshot_id));
/** @param {Record<string, unknown>} body */
const isSavedExportSource = (body) =>
body.source_revision_id === null &&
(body.source_fingerprint == null || typeof body.source_fingerprint === "string") &&
isOptionalSnapshot(body.snapshot);
/** @param {unknown} body */
function isSavedExportBody(body) {
if (!isJsonObject(body)) return false;
return (
isSavedId(body.client_operation_id) &&
typeof body.title_snapshot === "string" &&
[body.coordinate_width, body.coordinate_height, body.duration_seconds].every(Number.isFinite) &&
isJsonObject(body.render_settings) &&
isSavedUpload(body.local_upload) &&
isSavedExportSource(body)
);
}
/** @param {unknown} value @returns {value is NonNullable<VersionRecord["pending"]>} */
const isPending = (value) =>
isJsonObject(value) &&
isSavedExportBody(value.body) &&
isNullableSavedId(value.videoId) &&
isOptionalSavedId(value.projectId) &&
(value.checkingSince === undefined || Number.isFinite(value.checkingSince)) &&
isOptionalBoolean(value.snapshotRefused) &&
isOptionalBoolean(value.unanswered) &&
isOptionalReviewSent(value.reviewSent);
/** @param {unknown} sent */
const isOptionalReviewSent = (sent) =>
sent === undefined ||
(isJsonObject(sent) &&
Number.isFinite(sent.since) &&
(sent.value === undefined || sent.value === "yes" || sent.value === "no"));
/** @typedef {Omit<NonNullable<VersionRecord["last"]>, "fingerprint"> & {fingerprint?: string}} SavedLast */
/** @param {unknown} value @returns {value is SavedLast} */
function isLast(value) {
return (
isJsonObject(value) &&
(value.fingerprint === undefined || typeof value.fingerprint === "string") &&
isSavedId(value.versionId) &&
isSavedId(value.exportedAt) &&
isSavedId(value.projectId) &&
isLastMetadata(value)
);
}
/** @param {Record<string, unknown>} value */
function isLastMetadata(value) {
return (
isOptionalBoolean(value.files) &&
isOptionalSnapshot(value.snapshot) &&
(value.checksum === undefined || isSavedId(value.checksum))
);
}
/** @param {Record<string, unknown>} saved
* @returns {saved is {pending?: VersionRecord["pending"], last?: SavedLast | null, failure?: unknown}} */
const isExportState = (saved) =>
("pending" in saved || "last" in saved) &&
(saved.pending == null || isPending(saved.pending)) &&
(saved.last == null || isLast(saved.last));
/** @param {unknown} value @returns {value is Failure & { projectId: string }} */
const isSavedFailure = (value) =>
isJsonObject(value) &&
isSavedId(value.projectId) &&
typeof value.exportKey === "string" &&
typeof value.sentence === "string" &&
typeof value.final === "boolean" &&
typeof value.exportHelps === "boolean" &&
isOptionalBoolean(value.held);
/** @param {string} dir @returns {Promise<VersionRecord>} */
async function readState(dir) {
const file = statePath(dir);
const saved = await readJsonRecord(file);
if (!saved) return { pending: null, last: null };
if (!isExportState(saved)) throw new SharingStateError(file);
const { last, pending } = saved;
return {
pending: pending ?? null,
failure: isSavedFailure(saved.failure) ? saved.failure : null,
last: typeof last?.fingerprint === "string" ? { ...last, fingerprint: last.fingerprint } : null,
};
}
/** @param {string} dir @param {VersionRecord} state */
async function writeState(dir, state) {
await mkdirInside(dir, ".hyperframes");
await writeJsonAtomic(statePath(dir), state);
}
/** @typedef {{ sentence: string, final: boolean, exportHelps: boolean, held?: boolean }} Said
* @typedef {Said & { exportKey: string }} Failure */
/** @param {unknown} error @returns {Said} */
const said = (error) => ({
sentence: versionSentence(error),
final: error instanceof ReviewFailed && (FINAL.has(error.reason) || TOO_BIG.has(error.reason)),
exportHelps: error instanceof ReviewFailed && TOO_BIG.has(error.reason),
...(error instanceof ReviewFailed && error.reason === "held_for_review" && { held: true }),
});
/** A failure found here, before the server: fixing the export fixes it. @param {string} sentence @returns {Said} */
const local = (sentence) => ({ sentence, final: false, exportHelps: false });
/** @type {Map<string, ReviewExportProgress>} */
const inFlight = new Map();
/** A running watch's wait, woken so its next read is now. @type {Map<string, () => void>} */
const wakers = new Map();
/** Runs woken while not waiting (a read was out): their next wait is skipped. @type {Set<string>} */
const woken = new Set();
/** The last run's sentence, kept only for the export it was about: a new export starts clean.
* @type {Map<string, Failure>} */
const failures = new Map();
/** Which export a failure was about: the newest export file and the project as it was rendered. @param {string} dir */
const exportKeyOf = (dir) => {
const latest = latestExportFile(dir);
return latest ? `${latest.path}\n${latest.fingerprint ?? ""}\n${latest.id}` : "";
};
/** Whether the newest export can go up now: there is one, it is MP4, and the project has not moved past it.
* @param {string} dir @param {string} now @returns {ReviewVersion["source"]} */
function sourceOf(dir, now) {
const latest = latestExportFile(dir);
if (!latest) return "none";
if (!/\.mp4$/i.test(latest.path)) return "not-mp4";
return latest.fingerprint === now ? "ready" : "stale";
}
/** What the popover shows about versions. `last.current`: the project as it is now and, when the server says, still
* its latest ready version (a removed one, or a newer one from elsewhere, is not "on the page").
* @param {string} projectPath @param {{ latestReadyVersionId?: string | null }} [server] */
export async function reviewVersion(projectPath, server = {}) {
const [state, now, projectId] = await Promise.all([
readState(projectPath),
projectFingerprint(projectPath),
reviewProjectId(projectPath),
]);
const key = normalize(projectPath);
const last = state.last?.projectId === projectId ? state.last : null;
const onServer =
server.latestReadyVersionId === undefined || server.latestReadyVersionId === last?.versionId;
const savedFailure = state.failure?.projectId === projectId ? state.failure : null;
const failed = savedFailure ?? failures.get(key);
return {
last: last
? { exportedAt: last.exportedAt, current: last.fingerprint === now && onServer }
: null,
source: sourceOf(projectPath, now),
exporting: inFlight.get(key) ?? null,
...(failed?.exportKey === exportKeyOf(projectPath)
? {
failure: failed.sentence,
retryable: !failed.final,
exportHelps: failed.exportHelps,
...(failed.held && { held: true }),
}
: { failure: null }),
};
}
/** The newest MP4 and the body that adds it; a stale, non-MP4, too big or too long export never sends anything.
* @param {string} dir @param {string} title @param {(file: string) => Promise<VideoFacts | null>} probe
* @param {FilesForRender} filesForExport @param {boolean} requireSource */
async function exportSource(dir, title, probe, filesForExport, requireSource) {
const latest = latestExportFile(dir);
if (!latest || sourceOf(dir, await projectFingerprint(dir)) !== "ready")
return { failure: local("Export an MP4 of the project as it is now first.") };
const { size } = await stat(latest.path);
if (size > MAX_BYTES) return { failure: said(new ReviewFailed(400, "export_too_large")) };
const facts = await probe(latest.path);
if (!facts) return { failure: local("Couldn't read this video. Export it again.") };
if (facts.duration > MAX_SECONDS)
return { failure: said(new ReviewFailed(400, "export_too_long")) };
/** @type {ExportRequest} */
const body = {
client_operation_id: `export-${randomUUID()}`,
title_snapshot: title.slice(0, 255),
coordinate_width: facts.width,
coordinate_height: facts.height,
duration_seconds: facts.duration,
render_settings: {},
source_revision_id: null,
// The project as rendered, which is what the version shows; never "now".
source_fingerprint: latest.fingerprint ?? null,
local_upload: {
size_bytes: size,
checksum_sha256: await fileSha256Base64(latest.path),
content_type: "video/mp4",
},
};
const snapshot = await sourceSnapshot(
dir,
body.source_fingerprint,
filesForExport,
requireSource,
);
if (missingRequiredSource(requireSource, snapshot))
return {
failure: local(
"Source changed after preparation. Retry sharing, or choose video-only explicitly.",
),
};
if (snapshot) body.snapshot = snapshot;
return { file: latest.path, body };
}
/** @param {boolean} required @param {import("./types/reviewWire.js").SnapshotRef | null} snapshot */
const missingRequiredSource = (required, snapshot) => required && !snapshot;
/** @param {string} dir @param {string | null} rendered @param {FilesForRender} filesForExport @param {boolean} requireSource */
async function sourceSnapshot(dir, rendered, filesForExport, requireSource) {
return filesForExport(dir, rendered).catch((error) => {
if (requireSource) throw error;
console.error("[review files] could not read source; adding video only:", error);
return null;
});
}
/** @param {ExportRequest["snapshot"]} a @param {ExportRequest["snapshot"]} b */
const sameSnapshot = (a, b) =>
a?.space_id === b?.space_id &&
a?.composition_id === b?.composition_id &&
a?.snapshot_id === b?.snapshot_id;
/** A retry of the same file and project sends the pending body and its key unchanged. The snapshot is compared only
* while the server holds nothing: once it answered with a video, refused the snapshot, or may have made the video
* with no answer back, a different snapshot now would make one render two versions. A pending export of another
* server's project (the other backend's) is never resumed here; one saved before projectId was kept is this one's.
* @param {VersionRecord} state @param {ExportRequest} fresh @param {string} projectId */
function bodyToSend(state, fresh, projectId) {
const pending = state.pending;
if (pending && sameRender(pending, fresh, projectId) && reusableSnapshot(pending, fresh))
return pending;
return { body: fresh, videoId: null, projectId };
}
/** @param {NonNullable<VersionRecord["pending"]>} pending @param {ExportRequest} fresh @param {string} projectId */
function sameRender(pending, fresh, projectId) {
return (
(pending.projectId ?? projectId) === projectId &&
(pending.body.source_fingerprint ?? null) === fresh.source_fingerprint &&
pending.body.local_upload.checksum_sha256 === fresh.local_upload.checksum_sha256
);
}
/** @param {NonNullable<VersionRecord["pending"]>} pending @param {ExportRequest} fresh */
function reusableSnapshot(pending, fresh) {
if (pending.videoId || pending.snapshotRefused || pending.unanswered) return true;
return sameSnapshot(pending.body.snapshot, fresh.snapshot);
}
/** No answer, or a server error, can follow a create the server made; a refusal says it made nothing.
* @param {unknown} error */
const unanswered = (error) =>
!(error instanceof ReviewFailed) ||
error.status === 0 ||
error.status === 408 ||
error.status >= 500;
/** The export as the server has it. A 422 `snapshot_mismatch` never costs the version: it goes up as video only, under
* a new key when no earlier send is unanswered.
* @param {Required<VersionRequest>} r @param {string} id @param {NonNullable<VersionRecord["pending"]>} pending
* @param {(next: NonNullable<VersionRecord["pending"]>) => Promise<void>} kept */
async function createdExport(r, id, pending, kept) {
if (pending.videoId) return { dto: await r.client.readExport(id, pending.videoId), pending };
try {
return await sendCreate(r, id, pending, kept);
} catch (error) {
const refused = error instanceof ReviewFailed && error.reason === "snapshot_mismatch";
if (!refused || !pending.body.snapshot || pending.unanswered || r.requireSource) throw error;
console.error("[review] snapshot refused; the version goes up without files:", error);
const body = { ...pending.body, client_operation_id: `export-${randomUUID()}` };
delete body.snapshot;
const next = { ...pending, body, snapshotRefused: true };
await kept(next);
return sendCreate(r, id, next, kept);
}
}
/** @param {Required<VersionRequest>} r @param {string} id @param {NonNullable<VersionRecord["pending"]>} pending
* @param {(next: NonNullable<VersionRecord["pending"]>) => Promise<void>} kept */
async function sendCreate(r, id, pending, kept) {
const host = r.fromReview;
const { from_review: _, ...body } = pending.body;
const replay = pending.unanswered;
/** @param {ReviewSent} [reviewSent] */
const beforeSend = (reviewSent) => kept({ ...pending, unanswered: true, reviewSent });
/** @type {ReviewSent | undefined} */
let mark;
try {
if (!host && !replay) await beforeSend();
const dto = await r.client.createExport(
id,
host
? async () => {
mark = await readFromReview(host, r.projectPath);
// The auth loop only prepares a second send after a 401. A replay of an earlier run
// cannot assume that earlier send was refused.
if (!replay) await beforeSend({ since: mark.since });
return { ...body, ...(mark.value && { from_review: mark.value }) };
}
: pending.body,
);
const reviewSent = replay ? cleanupOnly(pending.reviewSent) : mark;
return { dto, pending: { ...pending, unanswered: false, reviewSent } };
} catch (error) {
if (!replay && !unanswered(error)) await kept(pending);
throw error;
}
}
/** @param {ReviewSent | undefined} mark */
const cleanupOnly = (mark) => mark && { since: mark.since };
/** Bytes sent, reported when the whole percent moves, not per chunk (a 2 GiB file is ~32k chunks).
* @param {number} size @param {(progress: ReviewExportProgress) => void} report */
function uploadReporter(size, report) {
let shown = -1;
return (/** @type {number} */ sent) => {
const percent = Math.floor((sent / Math.max(size, 1)) * 100);
if (percent !== shown) {
shown = percent;
report({ step: "uploading", sent, total: size });
}
};
}
/** Upload (once more with fresh instructions if the signed PUT expired), then complete.
* @param {Required<VersionRequest>} r @param {string} id @param {ExportDTO} created @param {string} file
* @param {number} size @param {(progress: ReviewExportProgress) => void} report @returns {Promise<ExportDTO>} */
async function upload(r, id, created, file, size, report) {
let current = created;
for (let tries = 0; current.status === "upload_pending" && current.upload && tries < 2; tries++) {
const sent = await r.put(current.upload, file, size, uploadReporter(size, report));
if (sent === "offline") throw new ReviewFailed(0, "offline");
if (sent === "refused") throw new ReviewFailed(502, "storage_refused");
if (sent === "sent") return r.client.completeExport(id, current.video_id);
current = await r.client.readExport(id, current.video_id);
}
// Two signed PUTs expired in a row: the slot is spent, and the next attempt starts over with a new key.
if (current.status === "upload_pending") throw new ReviewFailed(410, "upload_expired");
return current;
}
/** @param {ExportDTO} dto */
const going = (dto) => dto.status === "processing" || dto.status === "screening";
/** How long after the upload a screening read keeps the watch going: a day for a held video, twice the late mark for
* one still checking, and none past WATCH_MS for an older server that does not say. @param {ExportDTO} current */
const screeningWatch = (current) => {
if (current.screening_state === "held") return HELD_WATCH_MS;
return current.screening_state ? 2 * LATE_AFTER_MS : 0;
};
/** Watches processing and screening (2 s, 30 s once slow or held) until done or out of time; `since` and `keep` let
* a restart resume the 15-minute clock. @param {Required<VersionRequest>} r @param {string} id
* @param {ExportDTO} current @param {number} size @param {(progress: ReviewExportProgress) => void} report
* @param {{ since: number | null, keep: (since: number) => Promise<void> }} clock @returns {Promise<ExportDTO>} */
async function watch(r, id, current, size, report, clock) {
let until = r.now() + WATCH_MS;
let { since } = clock;
let checkingSince = Infinity;
while (going(current)) {
if (current.status === "screening") {
checkingSince = Math.min(checkingSince, r.now());
if (since === null) await clock.keep((since = r.now()));
until = Math.max(until, since + screeningWatch(current));
}
const progress = progressOf(
current,
size,
r.now() - (since ?? Infinity),
r.now() - checkingSince,
);
report(progress);
if (r.now() >= until || !r.open()) return current;
current = await poll(r, id, current, until, progress.slow ? SLOW_POLL_MS : POLL_MS);
}
return current;
}
/** One read as progress: late is counted from the upload, slow from this run's first screening read.
* @param {ExportDTO} current @param {number} size @param {number} sinceUpload @param {number} sinceChecking
* @returns {ReviewExportProgress} */
function progressOf(current, size, sinceUpload, sinceChecking) {
const checking = current.status === "screening";
const screening = checking ? current.screening_state : null;
const late = screening === "checking" && sinceUpload >= LATE_AFTER_MS;
/** @type {ReviewExportProgress} */
const progress = {
step: checking ? "checking" : "processing",
sent: size,
total: size,
slow: screening === "held" || sinceChecking >= SLOW_AFTER_MS,
};
if (screening) progress.screening = screening;
if (late) progress.late = true;
return progress;
}
/** Where checking is counted from: kept by an earlier run, else the upload completing in this one, kept with it.
* @param {Required<VersionRequest>} r @param {number | null} kept @param {boolean} completedNow
* @param {(since: number) => Promise<void>} keep */
async function checkingClock(r, kept, completedNow, keep) {
if (kept === null && completedNow) await keep((kept = r.now()));
return { since: kept, keep };
}
/** What a run says of a video its watch left still going. @param {ExportDTO} done */
const stillGoing = (done) =>
said(
new ReviewFailed(
202,
done.status === "screening" && done.screening_state === "held"
? "held_for_review"
: "still_checking",
),
);
/** A transient answer: busy, a server error, or no connection (a laptop waking up). @param {unknown} error */
const transient = (error) =>
error instanceof ReviewFailed &&
(error.status === 0 || error.status === 429 || error.status >= 500);
/** The next read of `current`; a failed one is retried while the watch lasts and its project is open, else the
* video comes back as it was last read.
* @param {Required<VersionRequest>} r @param {string} id @param {ExportDTO} current @param {number} until
* @param {number} every @returns {Promise<ExportDTO>} */
async function poll(r, id, current, until, every) {
const key = normalize(r.projectPath);
for (let wait = every; ; ) {
if (!woken.delete(key))
await new Promise((done) => {
wakers.set(key, () => done(undefined));
void r.sleep(wait).then(done);
});
wakers.delete(key);
try {
return await r.client.readExport(id, current.video_id);
} catch (error) {
if (!transient(error) || r.now() >= until) throw error;
if (!r.open()) return current;
wait = Math.min(
MAX_WAIT_MS,
Math.max(every, /** @type {ReviewFailed} */ (error).retryAfterMs ?? 0),
);
}
}
}
/** A refusal the same body would get again, or a spent upload, needs a new key next time.
* A refusal of a replay cannot settle an earlier unanswered create.
* @param {unknown} error @param {VersionRecord["pending"]} pending */
const spent = (error, pending) =>
(!pending?.unanswered || pending.videoId !== null) &&
error instanceof ReviewFailed &&
error.status >= 400 &&
error.status < 500 &&
![401, 408, 429].includes(error.status);
/** A video that is done: ready goes on the record as the newest version, anything else says why.
* @param {string} dir @param {VersionRecord} state @param {NonNullable<VersionRecord["pending"]>} pending
* @param {string} id @param {ExportDTO} done @param {FromReview | null} fromReview @returns {Promise<Said | null>} */
async function finish(dir, state, pending, id, done, fromReview) {
const ready = done.status === "ready";
const last = readyRecord(pending, id, done);
const failure = ready ? null : said(new ReviewFailed(200, done.error?.reason ?? ""));
await writeState(dir, {
pending: null,
last: ready ? last : state.last,
...(failure && {
failure: { ...failure, projectId: id, exportKey: await failedExportKey(dir, pending) },
}),
});
if (ready) await sharedFromReview(fromReview, dir, pending.reviewSent?.since);
return failure;
}
/** @param {string} dir @param {NonNullable<VersionRecord["pending"]>} pending */
async function failedExportKey(dir, pending) {
if (typeof pending.renderKey === "string") return pending.renderKey;
const latest = latestExportFile(dir);
if (!latest) return "";
const checksum =
latest.fingerprint === pending.body.source_fingerprint
? await fileSha256Base64(latest.path).catch(() => null)
: null;
return checksum === pending.body.local_upload.checksum_sha256
? exportKeyOf(dir)
: `pending:${pending.body.client_operation_id}`;
}
/** @param {Required<VersionRequest>} r @param {(progress: ReviewExportProgress) => void} report
* @returns {Promise<Said | null>} */
async function run(r, report) {
report({ step: "preparing", sent: 0, total: 0 });
const state = await readState(r.projectPath);
const source = await exportSource(
r.projectPath,
r.title,
r.probe,
r.omitSource ? noSource : r.filesForExport,
r.requireSource,
);
if ("failure" in source) return source.failure ?? null;
const id = r.projectId ?? (await reviewProjectId(r.projectPath));
if (!id) return local("Share the project first, then add a version.");
if (
sameSourceMode(r, state.last, source.body.snapshot) &&
(await alreadyUploaded(r.client, id, state, source.body))
)
return null;
return sendVersion(r, report, source, id, state);
}
const noSource = async () => null;
/** @param {Required<VersionRequest>} r @param {VersionRecord["last"]} last @param {ExportRequest["snapshot"]} snapshot */
function sameSourceMode(r, last, snapshot) {
if (r.requireSource) return sameSnapshot(last?.snapshot, snapshot);
if (r.omitSource) return last?.files !== true;
return true;
}
/** @param {ReviewClient} client @param {string} id @param {VersionRecord} state @param {ExportRequest} body */
async function alreadyUploaded(client, id, state, body) {
const last = state.last;
if (state.pending || state.failure?.projectId === id || !last || last.projectId !== id)
return false;
if (
last.fingerprint !== body.source_fingerprint ||
last.checksum !== body.local_upload.checksum_sha256
)
return false;
return (await client.project(id)).latest_ready_version_id === last.versionId;
}
/** @param {Required<VersionRequest>} r @param {(progress: ReviewExportProgress) => void} report
* @param {{file: string, body: ExportRequest}} source @param {string} id @param {VersionRecord} state */
async function sendVersion(r, report, source, id, state) {
const first = state.last?.projectId !== id;
let pending = bindRenderKey(bodyToSend(state, source.body, id), r.projectPath);
const sourceConflict = pendingSourceConflict(r, pending, source.body);
if (sourceConflict) return local(sourceConflict);
await writeState(r.projectPath, (state = { ...state, pending, failure: null }));
try {
const answer = await createdExport(r, id, pending, (next) =>
writeState(r.projectPath, (state = { ...state, pending: next })),
);
const created = answer.dto;
pending = answer.pending;
await writeState(
r.projectPath,
(state = { ...state, pending: { ...pending, videoId: created.video_id } }),
);
const size = pending.body.local_upload.size_bytes;
const uploaded = await upload(r, id, created, source.file, size, report);
if (uploaded.status !== "failed") await r.onUploaded();
const clock = await checkingClock(
r,
state.pending?.checkingSince ?? null,
created.status === "upload_pending" && going(uploaded),
async (since) => {
const kept = { ...pending, videoId: created.video_id, checkingSince: since };
await writeState(r.projectPath, (state = { ...state, pending: kept }));
},
);
const done = await watch(r, id, uploaded, size, report, clock);
return finishRun(r, state, pending, id, done, first);
} catch (error) {
const failure = said(error);
await writeState(r.projectPath, {
...state,
pending: spent(error, state.pending) ? null : state.pending,
failure: {
...failure,
projectId: id,
exportKey: await failedExportKey(r.projectPath, pending),
},
});
return failure;
}
}
/** @param {Required<VersionRequest>} r @param {NonNullable<VersionRecord["pending"]>} pending @param {ExportRequest} fresh */
function pendingSourceConflict(r, pending, fresh) {
if (r.requireSource && !pending.body.snapshot)
return "A pending video-only upload must finish before attaching source. Retry with --video-only, then share source again.";
if (r.omitSource && pending.body.snapshot)
return "A pending upload includes source. Check its status or retry with source before creating a video-only version.";
if (r.requireSource && !sameSnapshot(pending.body.snapshot, fresh.snapshot))
return "A pending upload includes a different source snapshot. Check its status before sharing again.";
return null;
}
/** @param {Required<VersionRequest>} r @param {VersionRecord} state
* @param {NonNullable<VersionRecord["pending"]>} pending @param {string} id @param {ExportDTO} done @param {boolean} first */
async function finishRun(r, state, pending, id, done, first) {
if (going(done)) return r.allowPending ? null : stillGoing(done);
const failed = await finish(r.projectPath, state, pending, id, done, r.fromReview);
if (!failed) {
const fromReview = pending.reviewSent?.value;
r.onAdded({ files: Boolean(pending.body.snapshot), first, ...(fromReview && { fromReview }) });
}
return failed;
}
/** Adds the newest export as a version and resolves once it is ready, failed, or out of watch, with the sentence when
* it did not go up. One run per project; a second asks nothing and resolves at once. A run onto another project keeps
* a failure from before the upload to itself: the recorded page is not the one it failed on.
* @param {VersionRequest} request @returns {Promise<string | null>} */
export async function addVersion(request) {
let uploaded = false;
const r = {
put: putFile,
sleep: (/** @type {number} */ ms) => new Promise((done) => setTimeout(done, ms)),
probe: probeVideo,
// Read through Date each time, so a walk can move main's clock under a run.
now: () => Date.now(),
onProgress: () => {},
open: () => true,
allowPending: false,
filesForExport: noSource,
fromReview: null,
requireSource: false,
omitSource: false,
onAdded: () => {},
...request,
projectId: request.projectId ?? null,
onUploaded: async () => {
uploaded = true;
await request.onUploaded?.();
},
};
const key = normalize(r.projectPath);
if (inFlight.has(key)) return null;
failures.delete(key);
/** @param {ReviewExportProgress} progress */
const report = (progress) => (inFlight.set(key, progress), r.onProgress(progress));
try {
report({ step: "preparing", sent: 0, total: 0 });
const failure = await withProjectShareLock(r.projectPath, "export", () => run(r, report));
const kept = failure && (uploaded || !request.projectId);
if (kept) failures.set(key, { ...failure, exportKey: exportKeyOf(r.projectPath) });
return failure?.sentence ?? null;
} catch (error) {
const sentence =
error instanceof SharingStateError ? error.message : "Couldn't add this version. Try again.";
failures.set(key, {
sentence,
final: false,
exportHelps: false,
exportKey: exportKeyOf(r.projectPath),
});
return sentence;
} finally {
inFlight.delete(key);
woken.delete(key);
r.onProgress(null);
}
}
/** The popover opening reads a held video again: a running watch reads now, and one that ran out reads the video
* back once, so a video staff approved or refused since shows as it is. A read that finds it still held changes
* nothing; Check again starts a new watch.
* @param {string} projectPath @param {ReviewClient} client @param {FromReview | null} [fromReview] */
export async function recheckHeld(projectPath, client, fromReview = null) {
const key = normalize(projectPath);
if (inFlight.has(key)) return void (wakers.get(key) ?? (() => woken.add(key)))();
const failed = failures.get(key);
if (!failed?.held) return;
const { pending } = await readState(projectPath);
const id = pending?.projectId ?? (await reviewProjectId(projectPath));
if (!pending?.videoId || !id) return;
const done = await client.readExport(id, pending.videoId);
if (going(done)) return;
await whenExportIdle(projectPath, async () => {
if (inFlight.has(key) || failures.get(key) !== failed) return;
const state = await readState(projectPath);
const current = state.pending;
if (
!current ||
current.videoId !== pending.videoId ||
current.body.client_operation_id !== pending.body.client_operation_id ||
(current.projectId ?? (await reviewProjectId(projectPath))) !== id
) {
return;
}
const failure = await finish(projectPath, state, current, id, done, fromReview);
if (failure) failures.set(key, { ...failure, exportKey: failed.exportKey });
else failures.delete(key);
});
}
/** @param {string} dir @param {() => Promise<void>} run */
async function whenExportIdle(dir, run) {
try {
await withProjectShareLock(dir, "export", run, { timeoutMs: 0 });
} catch (error) {
if (!(error instanceof ShareBusyError)) throw error;
}
}
/** Settle a saved upload after its host exits, without competing with an active uploader.
* @param {string} projectPath @param {ReviewClient} client @param {FromReview | null} [fromReview] */
export async function refreshReviewExport(projectPath, client, fromReview = null) {
await whenExportIdle(projectPath, () => settleSavedExport(projectPath, client, fromReview));
}
/** @param {string} dir @param {ReviewClient} client @param {FromReview | null} fromReview */
async function settleSavedExport(dir, client, fromReview) {
const state = await readState(dir);
const pending = state.pending;
const id = await reviewProjectId(dir);
if (!pending?.videoId || !id || pending.projectId !== id) return;
const done = await client.readExport(id, pending.videoId);
if (done.status === "upload_pending" || going(done)) return;
const failure = await finish(dir, state, pending, id, done, fromReview);
const key = normalize(dir);
if (failure) failures.set(key, { ...failure, exportKey: exportKeyOf(dir) });
else failures.delete(key);
}
/** The first share puts the newest export up with the link, when there is one to put and no version yet.
* @param {VersionRequest} request */
export async function addFirstVersion(request) {
const version = await reviewVersion(request.projectPath);
if (version.last || version.source !== "ready") return;
void addVersion(request);
}
/** What the files line and a later attach read of a project's versions: the newest one this app put on the page, with
* whether the project is still as it was then; whether the newest export is the project now; whether a run is out.
* @param {string} dir */
export async function versionFacts(dir) {
const [state, now, projectId] = await Promise.all([
readState(dir),
projectFingerprint(dir),
reviewProjectId(dir),
]);
const last = state.last?.projectId === projectId ? state.last : null;
return {
last: last && { ...last, current: last.fingerprint === now },
source: sourceOf(dir, now),
running: inFlight.has(normalize(dir)),
};
}
/** The newest version this app put on the page, read without hashing the project; null while a run is out.
* @param {string} dir */
export async function versionOnPage(dir) {
const [state, projectId] = await Promise.all([readState(dir), reviewProjectId(dir)]);
if (inFlight.has(normalize(dir))) return null;
return state.last?.projectId === projectId ? state.last : null;
}
/** Records that `versionId` carries files now, if it is still the last ready version.
* @param {string} dir @param {string} versionId @param {string} [projectId] */
export async function markFiles(dir, versionId, projectId) {
if (inFlight.has(normalize(dir))) return;
await withProjectShareLock(dir, "export", async () => {
const state = await readState(dir);
const { last } = state;
if (
!last ||
last.versionId !== versionId ||
(projectId !== undefined && last.projectId !== projectId)
) {
return;
}
await writeState(dir, { ...state, last: { ...last, files: true } });
});
}
/** A pending operation is not the last ready version. @param {string} projectPath */
export async function pendingReviewExport(projectPath) {
const { pending } = await readState(projectPath);
if (!pending) return null;
return {
projectId: pending.projectId ?? null,
videoId: pending.videoId,
operationId: pending.body.client_operation_id,
fingerprint: pending.body.source_fingerprint ?? null,
checksum: pending.body.local_upload.checksum_sha256,
};
}
/** @param {NonNullable<VersionRecord["pending"]>} pending @param {string} id @param {ExportDTO} done */
function readyRecord(pending, id, done) {
return {
fingerprint: pending.body.source_fingerprint ?? "",
versionId: done.version_id,
exportedAt: done.exported_at,
projectId: id,
files: Boolean(pending.body.snapshot),
...(pending.body.snapshot && { snapshot: pending.body.snapshot }),
checksum: pending.body.local_upload.checksum_sha256,
};
}
/** @param {NonNullable<VersionRecord["pending"]>} pending @param {string} dir */
function bindRenderKey(pending, dir) {
return typeof pending.renderKey === "string"
? pending
: { ...pending, renderKey: exportKeyOf(dir) };
}