diff --git a/packages/cli/src/commands/publish.test.ts b/packages/cli/src/commands/publish.test.ts index 92b7e0986e..bbaf15dea9 100644 --- a/packages/cli/src/commands/publish.test.ts +++ b/packages/cli/src/commands/publish.test.ts @@ -3,7 +3,21 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; -const publishState = vi.hoisted(() => ({ publish: vi.fn(), noticeRelease: null as string | null })); +const publishState = vi.hoisted(() => ({ + publish: vi.fn(), + noticeRelease: null as string | null, + confirm: vi.fn(), +})); + +vi.mock("../ui/prompts.js", async (importOriginal) => ({ + ...(await importOriginal()), + confirm: publishState.confirm, +})); + +vi.mock("../utils/attendedTerminal.js", async (importOriginal) => ({ + ...(await importOriginal()), + isUnattended: () => false, +})); vi.mock("../utils/publishProject.js", async (importOriginal) => ({ ...(await importOriginal()), @@ -190,6 +204,71 @@ describe("publish visibility messaging", () => { expect(output).not.toContain("Private — authentication and access required"); }); + it("asks before updating an existing project named by the folder, without promising a private upload", async () => { + const project = mkdtempSync(join(tmpdir(), "hf-publish-existing-")); + const config = mkdtempSync(join(tmpdir(), "hf-publish-existing-config-")); + writeFileSync(join(project, "index.html"), VISIBLE_INDEX); + writeTeamProject(project, { projectId: "existing-\u001b[2Kproject" }); + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", config); + vi.stubEnv("HEYGEN_API_KEY", ""); + vi.stubEnv("HYPERFRAMES_API_KEY", ""); + publishState.publish.mockReset(); + publishState.confirm.mockReset(); + publishState.confirm.mockResolvedValue(false); + const prompt = (isPublic: boolean) => + captureLog(() => + publishCommand.run?.({ args: { dir: project, public: isPublic, proxy: false } } as never), + ); + try { + await writeStore({ oauth: { access_token: "token" } }, join(config, "credentials")); + const output = await prompt(false); + expect(output).toContain( + "If you can access project existing-?[2Kproject, this updates it in place and keeps its current visibility, which may be public. Otherwise it creates a new private project.", + ); + expect(output).not.toContain("privately"); + expect(output).toContain("Aborted."); + const publicOutput = await prompt(true); + expect(publicOutput).toContain("requests public visibility"); + expect(publicOutput).toContain( + "If you can access project existing-?[2Kproject, this updates it in place. Otherwise it creates a new public project.", + ); + await writeStore({ api_key: "valid-key" }, join(config, "credentials")); + const signedOut = await prompt(false); + expect(signedOut).toContain("uploads this project privately"); + expect(signedOut).not.toContain("existing-"); + expect(publishState.publish).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + rmSync(project, { recursive: true, force: true }); + rmSync(config, { recursive: true, force: true }); + } + }); + + it("refuses --update without a login before asking", async () => { + const project = mkdtempSync(join(tmpdir(), "hf-publish-update-signed-out-")); + const config = mkdtempSync(join(tmpdir(), "hf-publish-update-signed-out-config-")); + writeFileSync(join(project, "index.html"), VISIBLE_INDEX); + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", config); + vi.stubEnv("HEYGEN_API_KEY", ""); + vi.stubEnv("HYPERFRAMES_API_KEY", ""); + publishState.confirm.mockReset(); + try { + consumeCommandResult(); + const output = await captureLog(() => + publishCommand.run?.({ + args: { dir: project, public: false, proxy: false, update: "target" }, + } as never), + ); + expect(output).toContain("--update requires authentication"); + expect(publishState.confirm).not.toHaveBeenCalled(); + expect(consumeCommandResult().exitCode).toBe(1); + } finally { + vi.unstubAllEnvs(); + rmSync(project, { recursive: true, force: true }); + rmSync(config, { recursive: true, force: true }); + } + }); + it("labels an authentication-required anonymous URL as a claim URL", async () => { const output = await runPublish({ public: false, claimed: false }); diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index ddc9e79f30..09438f4f26 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -146,35 +146,12 @@ export default defineCommand({ } } - if (args.yes !== true) { - console.log(); - if (args.public === true) { - console.log( - ` ${c.bold("hyperframes publish uploads this project and requests public visibility at a stable URL.")}`, - ); - console.log(` ${c.dim("Anyone with the URL can open a claimed public project.")}`); - } else { - console.log( - ` ${c.bold("hyperframes publish uploads this project privately to a stable URL.")}`, - ); - console.log( - ` ${c.dim("Viewing requires authentication and access. Pass --public to allow anyone with the URL.")}`, - ); - } - console.log(); - const approved = await clack.confirm({ message: "Publish this project?" }); - if (clack.isCancel(approved) || approved !== true) { - console.log(); - console.log(` ${c.dim("Aborted.")}`); - console.log(); - return; - } - } - const updateTarget = typeof args.update === "string" && args.update.trim() ? parseUpdateTarget(args.update) : undefined; + const committedTeam = readTeamProject(dir); + const priorLink = readProjectLink(dir); const spaceOverride = typeof args.space === "string" && args.space.trim() ? args.space.trim() : undefined; @@ -215,7 +192,48 @@ export default defineCommand({ } } - const committedTeam = readTeamProject(dir); + const existingTarget = + credential?.type === "oauth" + ? ( + updateTarget ?? + committedTeam?.projectId ?? + (priorLink?.url ? priorLink.projectId : undefined) + )?.replace(/[^!-~]/g, "?") + : undefined; + + if (args.yes !== true) { + console.log(); + if (args.public === true) { + console.log( + ` ${c.bold("hyperframes publish uploads this project and requests public visibility at a stable URL.")}`, + ); + console.log(` ${c.dim("Anyone with the URL can open a claimed public project.")}`); + } else if (!existingTarget) { + console.log( + ` ${c.bold("hyperframes publish uploads this project privately to a stable URL.")}`, + ); + console.log( + ` ${c.dim("Viewing requires authentication and access. Pass --public to allow anyone with the URL.")}`, + ); + } + if (existingTarget) { + const keeps = + args.public === true ? "" : " and keeps its current visibility, which may be public"; + const created = args.public === true ? "public" : "private"; + console.log( + ` ${c.bold(`If you can access project ${existingTarget}, this updates it in place${keeps}. Otherwise it creates a new ${created} project.`)}`, + ); + } + console.log(); + const approved = await clack.confirm({ message: "Publish this project?" }); + if (clack.isCancel(approved) || approved !== true) { + console.log(); + console.log(` ${c.dim("Aborted.")}`); + console.log(); + return; + } + } + // Stable id: explicit --update wins, else the committed team id, else this machine's stored/minted id. const requestedProjectId = updateTarget ?? committedTeam?.projectId ?? ensureProjectId(dir); // Team space: explicit --space wins, else the committed space id, else the personal space. @@ -223,7 +241,6 @@ export default defineCommand({ // Continuity cue: if this directory was published before, show where it lives so the // user knows a re-publish updates that same link (when logged in). - const priorLink = readProjectLink(dir); if (priorLink?.url) { console.log(); console.log(` ${c.dim(`Previously published at ${priorLink.url}`)}`); diff --git a/packages/cli/src/heygen/jobLedger.ts b/packages/cli/src/heygen/jobLedger.ts new file mode 100644 index 0000000000..017dcc85f4 --- /dev/null +++ b/packages/cli/src/heygen/jobLedger.ts @@ -0,0 +1,131 @@ +import { createHash } from "node:crypto"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + realpathSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { configDir } from "../auth/paths.js"; +import { Fix } from "../measure/media.js"; +import { creditsSpent } from "../media-use/lib/credit-limit.mjs"; + +// Each account's own record of what its jobs in a project folder hold, kept in its config folder and never in the +// project, so a job log row a sync or a pull brings in can neither lower it nor let a call skip the limit. +const ledgerFile = () => join(configDir(), "heygen-job-ledger"); +const LINES_KEPT = 10_000; + +type Kind = "own" | "seen"; +type Ledger = Record>; + +const toCredits = (value: unknown): number => + typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; + +/** Where ledger lines belong: this account and this project folder; null when either cannot be told. A moved or + * renamed folder starts a new ledger. */ +export function ledgerScope(scope: string | null, projectDir: string | undefined): string | null { + if (!scope || !projectDir) return null; + let real: string; + try { + real = realpathSync(projectDir); + } catch { + real = resolve(projectDir); + } + return `${scope}:${createHash("sha256").update(real).digest("hex").slice(0, 16)}`; +} + +function parse(line: string): Record | null { + try { + return Object(JSON.parse(line)); + } catch { + return null; + } +} + +function readLedger(at: string): Ledger { + const ledger: Ledger = { own: new Map(), seen: new Map() }; + const file = ledgerFile(); + if (!existsSync(file)) return ledger; + for (const line of readFileSync(file, "utf8").split("\n")) { + const row = parse(line); + if (!row) continue; + const { kind, key } = row; + if (row.at === at && typeof key === "string" && (kind === "own" || kind === "seen")) + ledger[kind].set(key, toCredits(row.credits)); + } + return ledger; +} + +function note(at: string, kind: Kind, entries: [string, number][]): void { + if (entries.length === 0) return; + mkdirSync(configDir(), { recursive: true, mode: 0o700 }); + const lines = entries.map(([key, credits]) => `${JSON.stringify({ at, kind, key, credits })}\n`); + appendFileSync(ledgerFile(), lines.join(""), { mode: 0o600 }); + const kept = readFileSync(ledgerFile(), "utf8").split("\n").filter(Boolean); + if (kept.length <= LINES_KEPT) return; + const temp = `${ledgerFile()}.${process.pid}.tmp`; + writeFileSync(temp, `${trimmed(kept, at).join("\n")}\n`, { mode: 0o600 }); + renameSync(temp, ledgerFile()); +} + +/** The newest lines up to LINES_KEPT, always with the last line `at` holds for each key. */ +function trimmed(lines: string[], at: string): string[] { + const last = new Map(); + const live = lines.map((line, n) => { + const row = parse(line); + if (row?.at === at) last.set(`${String(row.kind)} ${String(row.key)}`, n); + return row?.at === at; + }); + const keep = new Set(last.values()); + let room = LINES_KEPT - keep.size; + for (let n = lines.length - 1; n >= 0 && room > 0; n--) + if (!live[n]) { + keep.add(n); + room--; + } + return lines.filter((_, n) => keep.has(n)); +} + +export function noteOwn(at: string | null, key: string, credits: number): void { + if (at) note(at, "own", [[key, toCredits(credits)]]); +} + +export interface Spent { + total: number; + own: Map; + others: Map; + /** Credits counted from other keys' highest seen values above what the job log holds now. */ + gone: number; +} + +/** What a project's jobs hold: this account's keys at what it noted, any other key in the job log at the most it was + * seen to hold. A new highest for another key is noted. With no account to read it for, nothing is trusted. */ +export function spentIn(projectDir: string, at: string | null): Spent { + if (!at) + throw new Fix( + "can't tell which account is signed in, so the limit can't be checked; nothing was made", + "run the same command again; if this comes back, npx hyperframes auth login", + ); + const ledger = readLedger(at); + const logged = creditsSpent(projectDir).keys; + const others = new Map(ledger.seen); + for (const [key, credits] of logged) others.set(key, Math.max(credits, others.get(key) ?? 0)); + for (const key of ledger.own.keys()) others.delete(key); + note( + at, + "seen", + [...others].filter(([key, credits]) => credits > (ledger.seen.get(key) ?? 0)), + ); + const total = [...ledger.own.values(), ...others.values()].reduce((a, b) => a + b, 0); + let gone = 0; + for (const [key, credits] of others) gone += credits - (logged.get(key) ?? 0); + return { total, own: ledger.own, others, gone }; +} + +export const goneNote = (spent: Spent): string => + spent.gone > 0 + ? `. ${spent.gone} of the credits counted are from job log rows no longer in the project, noted in ${ledgerFile()}` + : ""; diff --git a/packages/cli/src/heygen/v3.test.ts b/packages/cli/src/heygen/v3.test.ts index de97cd92ce..630e23c098 100644 --- a/packages/cli/src/heygen/v3.test.ts +++ b/packages/cli/src/heygen/v3.test.ts @@ -1,4 +1,5 @@ import { + appendFileSync, linkSync, lstatSync, mkdirSync, @@ -184,6 +185,124 @@ describe("runJob", () => { await expect(runJob(X, {}, "k2", { log, credits: 100 })).rejects.toThrow(/leaves 0/); }); + it("counts a logged row of negative or non-finite credits as nothing", async () => { + const calls = sent(); + const log = project(100); + writeFileSync( + join(log.projectDir, ".hyperframes/heygen-jobs.jsonl"), + '{"key":"a","credits":-1000}\n{"key":"b","credits":-1e999}\n{"key":"c","credits":40}\n', + ); + await expect(runJob(X, {}, "k1", { log, credits: 61 })).rejects.toThrow(/leaves 60/); + expect(calls).toHaveLength(0); + }); + + it("checks a logged key this account never sent as a new call", async () => { + const calls = sent(); + const log = project(100); + writeFileSync( + join(log.projectDir, ".hyperframes/heygen-jobs.jsonl"), + '{"key":"k1","status":"completed","credits":1}\n', + ); + await expect(runJob(X, {}, "k1", { log, credits: 150 })).rejects.toThrow(/leaves 100/); + expect(calls).toHaveLength(0); + await runJob(X, {}, "k1", { log, credits: 100 }); + expect(calls).toHaveLength(1); + }); + + it("resumes a job this account sent unchecked, though its estimate is above what it was charged", async () => { + const log = project(100); + const calls: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + calls.push(url); + const charged = { amount: 50, pool: "hyperframes_credits" }; + return reply(200, { data: { id: "j", status: "completed", charged } }); + }); + await runJob(X, {}, "k1", { log, credits: 60 }); + await runJob(X, {}, "k2", { log, credits: 50 }); + await runJob(X, {}, "k1", { log, credits: 60 }); + expect(calls).toHaveLength(3); + }); + + it("checks a key this account sent for another project as a new call here", async () => { + const calls = sent(); + await runJob(X, {}, "k1", { log: project(100), credits: 10 }); + const log = project(100); + writeFileSync( + join(log.projectDir, ".hyperframes/heygen-jobs.jsonl"), + '{"key":"k1","status":"completed","credits":1}\n', + ); + await expect(runJob(X, {}, "k1", { log, credits: 150 })).rejects.toThrow(/leaves 100/); + expect(calls).toHaveLength(1); + }); + + it("keeps the account's ledger private, its newest lines, and every key this project holds", async () => { + sent(); + const file = join(process.env["HYPERFRAMES_CONFIG_DIR"]!, "heygen-job-ledger"); + const log = project(100); + await runJob(X, {}, "k0", { log, credits: 10 }); + const old = (n: number) => + `${JSON.stringify({ at: "old", kind: "own", key: `k${n}`, credits: 1 })}\n`; + appendFileSync(file, Array.from({ length: 10_000 }, (_, n) => old(n)).join("")); + await runJob(X, {}, "k1", { log, credits: 10 }); + const rows = readFileSync(file, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(rows).toHaveLength(10_000); + expect(rows.filter((row) => row.at !== "old").map((row) => row.key)).toEqual(["k0", "k1"]); + expect(rows.at(-1)).toMatchObject({ kind: "own", key: "k1", credits: 10 }); + await expect(runJob(X, {}, "k2", { log, credits: 81 })).rejects.toThrow(/leaves 80/); + if (process.platform === "win32") return; + const fresh = mkdtempSync(join(tmpdir(), "hf-v3-config-")); + vi.stubEnv("HYPERFRAMES_CONFIG_DIR", join(fresh, "nested")); + await runJob(X, {}, "k1", { log: project(100), credits: 10 }); + expect(lstatSync(join(fresh, "nested")).mode & 0o777).toBe(0o700); + expect(lstatSync(join(fresh, "nested", "heygen-job-ledger")).mode & 0o777).toBe(0o600); + }); + + it("names the ledger when rows gone from the job log still count", async () => { + sent(); + const log = project(100); + const file = join(log.projectDir, ".hyperframes/heygen-jobs.jsonl"); + writeFileSync(file, '{"key":"other","credits":60}\n'); + await runJob(X, {}, "k1", { log, credits: 10 }); + writeFileSync(file, ""); + await expect(runJob(X, {}, "k2", { log, credits: 40 })).rejects.toThrow( + /leaves 30.*60 of the credits counted are from job log rows no longer in the project, noted in .*heygen-job-ledger/, + ); + }); + + it("refuses a call under a limit when the signed-in account can't be named", async () => { + vi.stubEnv("HEYGEN_API_KEY", ""); + vi.stubEnv("HYPERFRAMES_API_KEY", ""); + vi.stubEnv("HEYGEN_ACCESS_TOKEN", "token"); + const calls: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + if (String(url).includes("/users/me")) return reply(500, { error: "down" }); + calls.push(url); + return reply(200, { data: { id: "j", status: "completed" } }); + }); + await expect(runJob(X, {}, "k1", { log: project(100), credits: 10 })).rejects.toThrow( + "can't tell which account is signed in, so the limit can't be checked; nothing was made", + ); + expect(calls).toHaveLength(0); + const unlimited = mkdtempSync(join(tmpdir(), "hf-v3-unlimited-")); + await runJob(X, {}, "k1", { log: { projectDir: unlimited, what: "a line" }, credits: 10 }); + expect(calls).toHaveLength(1); + }); + + it("keeps what this account spent, and the most another key held, when rows are added to the log", async () => { + const calls = sent(); + const log = project(100); + await runJob(X, {}, "k1", { log, credits: 30 }); + const file = join(log.projectDir, ".hyperframes/heygen-jobs.jsonl"); + appendFileSync(file, '{"key":"other","credits":40}\n'); + await expect(runJob(X, {}, "k2", { log, credits: 31 })).rejects.toThrow(/leaves 30/); + appendFileSync(file, '{"key":"k1","credits":0}\n{"key":"other","credits":0}\n'); + await expect(runJob(X, {}, "k2", { log, credits: 31 })).rejects.toThrow(/leaves 30/); + expect(calls).toHaveLength(1); + }); + it("lets the same job resume, and gives back a refused call's credits", async () => { const log = project(100); const calls = sent(); diff --git a/packages/cli/src/heygen/v3.ts b/packages/cli/src/heygen/v3.ts index 7db3a3f595..134812463f 100644 --- a/packages/cli/src/heygen/v3.ts +++ b/packages/cli/src/heygen/v3.ts @@ -11,7 +11,8 @@ import { refreshIfNeeded, resolveCloudAuthHeaders, resolveCloudBaseUrl } from ". import { HyperframesApiError, HyperframesCloudClient } from "../cloud/_gen/client.js"; import { Fix } from "../measure/media.js"; import { appendRecord } from "../utils/projectRecords.js"; -import { creditLimit, creditsSpent } from "../media-use/lib/credit-limit.mjs"; +import { creditLimit } from "../media-use/lib/credit-limit.mjs"; +import { goneNote, ledgerScope, noteOwn, spentIn } from "./jobLedger.js"; import { cliCheckoutLink, pageUrl } from "./plan.js"; const MEDIA_USE_SKILL = "media-use"; @@ -88,23 +89,24 @@ async function jobSender(): Promise { return { auth: buildAuthHeaders(credential), scope }; } -function failedKeys(scope: string): Set { - const file = failedKeysFile(); +function scopedKeys(file: string, scope: string): Set { const lines = existsSync(file) ? readFileSync(file, "utf8").split("\n") : []; return new Set( lines.filter((l) => l.startsWith(`${scope} `)).map((l) => l.slice(scope.length + 1)), ); } -function markFailed(key: string, scope: string | null): void { +function noteKey(file: string, key: string, scope: string | null): void { if (!scope) return; - mkdirSync(configDir(), { recursive: true }); - appendFileSync(failedKeysFile(), `${scope} ${key}\n`); + mkdirSync(configDir(), { recursive: true, mode: 0o700 }); + appendFileSync(file, `${scope} ${key}\n`, { mode: 0o600 }); } +const markFailed = (key: string, scope: string | null) => noteKey(failedKeysFile(), key, scope); + /** The key to send: `key` itself, or past every key whose job failed or was refused, so a retry is a new job. */ function liveKey(key: string, scope: string | null): string { - const failed = scope ? failedKeys(scope) : new Set(); + const failed = scope ? scopedKeys(failedKeysFile(), scope) : new Set(); let sent = key; for (let n = 1; failed.has(sent); n++) sent = `${key}-${n}`; return sent; @@ -254,8 +256,11 @@ function appendJobLog(projectDir: string, line: Record): void { ); } -function log(entry: JobLog, record: Record) { - if (entry.projectDir) appendJobLog(entry.projectDir, { what: entry.what, ...record }); +function log(entry: JobLog, record: Record, at: string | null) { + if (!entry.projectDir) return; + appendJobLog(entry.projectDir, { what: entry.what, ...record }); + if (typeof record.key === "string" && typeof record.credits === "number") + noteOwn(at, record.key, record.credits); } export interface RunOptions { @@ -271,19 +276,20 @@ export interface RunOptions { /** Refuses, before anything is sent, a new job past the project's credit limit, else reserves its credits. A vendor-priced * image is checked and held at its estimate plus 10%, which its charge can pass, until HeyGen says what it charged. - * Every paid call comes through runJob. True when this call reserved. */ + * A key resumes unchecked only if this account's ledger holds credits for it, else is priced as new. True when reserved. */ function withinLimit( limitLog: JobLog | undefined, sent: string, + at: string | null, credits: number | undefined, vendorPriced?: boolean, ): boolean { if (!limitLog?.projectDir) return false; const limit = creditLimit(limitLog.projectDir); if (limit === undefined) return false; - const spent = creditsSpent(limitLog.projectDir); - if (spent.keys.has(sent)) return false; - const left = limit - spent.total; + const spent = spentIn(limitLog.projectDir, at); + if ((spent.own.get(sent) ?? 0) > 0) return false; + const left = limit - spent.total + (spent.others.get(sent) ?? 0); const fix = "tell the person what their limit leaves out, then ask whether to raise it or make the video without this piece"; if (credits === undefined) @@ -294,10 +300,10 @@ function withinLimit( const { checked, about } = limitCheck(credits, vendorPriced); if (checked > left) throw new Fix( - `this call is ${about} and the run's credit limit of ${limit} leaves ${Math.max(0, left)}; nothing was made or charged`, + `this call is ${about} and the run's credit limit of ${limit} leaves ${Math.max(0, left)}; nothing was made or charged${goneNote(spent)}`, fix, ); - log(limitLog, { key: sent, status: "reserved", credits: checked }); + log(limitLog, { key: sent, status: "reserved", credits: checked }, at); return true; } @@ -313,7 +319,8 @@ export async function runJob( const route = jobRoute.path; const { auth, scope } = await jobSender(); const sent = liveKey(key, scope); - const reserved = withinLimit(opts.log, sent, opts.credits, opts.vendorPriced); + const at = ledgerScope(scope, opts.log?.projectDir); + const reserved = withinLimit(opts.log, sent, at, opts.credits, opts.vendorPriced); const held = opts.credits === undefined ? undefined : limitCheck(opts.credits, opts.vendorPriced).checked; let job: Job; @@ -328,13 +335,13 @@ export async function runJob( } catch (err) { if (err instanceof Refused && !KEEPS_KEY.has(err.status)) { markFailed(sent, scope); - if (reserved && opts.log) log(opts.log, { key: sent, status: "refused", credits: 0 }); + if (reserved && opts.log) log(opts.log, { key: sent, status: "refused", credits: 0 }, at); } throw err; } const waits = RUNNING.has(job.status.toLowerCase()); if (opts.log && waits) - log(opts.log, { route, id: job.id, key: sent, status: job.status, credits: held }); + log(opts.log, { route, id: job.id, key: sent, status: job.status, credits: held }, at); const status = jobRoute.status; while (status && RUNNING.has(job.status.toLowerCase())) { if (Date.now() - started > (opts.timeoutMs ?? 15 * 60_000)) @@ -352,13 +359,17 @@ export async function runJob( ); } if (opts.log) - log(opts.log, { - route, - id: job.id, - key: sent, - status: job.status, - credits: chargedCredits(job) ?? opts.credits, - }); + log( + opts.log, + { + route, + id: job.id, + key: sent, + status: job.status, + credits: chargedCredits(job) ?? opts.credits, + }, + at, + ); if (!/^(completed|succeeded|success|done)$/i.test(job.status)) { markFailed(sent, scope); throw await refusal(422, { diff --git a/packages/cli/src/media-use/lib/credit-limit.d.mts b/packages/cli/src/media-use/lib/credit-limit.d.mts index 9423599bba..52e6325967 100644 --- a/packages/cli/src/media-use/lib/credit-limit.d.mts +++ b/packages/cli/src/media-use/lib/credit-limit.d.mts @@ -1,3 +1,3 @@ export function creditLimit(projectDir: string): number | undefined; -export function creditsSpent(projectDir: string): { total: number; keys: Set }; +export function creditsSpent(projectDir: string): { total: number; keys: Map }; export function refuseUnderLimit(projectDir: string, what: string, generateType: string): void; diff --git a/packages/cli/src/media-use/lib/credit-limit.mjs b/packages/cli/src/media-use/lib/credit-limit.mjs index 02e44c6d5f..fa7b6b5ff7 100644 --- a/packages/cli/src/media-use/lib/credit-limit.mjs +++ b/packages/cli/src/media-use/lib/credit-limit.mjs @@ -16,18 +16,23 @@ export function creditLimit(projectDir) { return credits; } -/** Credits already committed by jobs in this project, and the job keys behind them. */ +/** Credits already committed by jobs in this project, and what each job key holds. A logged amount that is not a + * positive finite number holds nothing, so no row counts below zero. */ export function creditsSpent(projectDir) { const charged = new Map(); - for (const row of read(join(projectDir, ".hyperframes", "heygen-jobs.jsonl")).split("\n")) { - if (!row) continue; - const { key, credits } = JSON.parse(row); - if (key && typeof credits === "number") charged.set(key, credits); + for (const line of read(join(projectDir, ".hyperframes", "heygen-jobs.jsonl")).split("\n")) { + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + const { key, credits } = Object(row); + if (key && typeof credits === "number") + charged.set(key, Number.isFinite(credits) && credits > 0 ? credits : 0); } - return { - total: [...charged.values()].reduce((a, b) => a + b, 0), - keys: new Set([...charged].filter(([, c]) => c > 0).map(([k]) => k)), - }; + const keys = new Map([...charged].filter(([, c]) => c > 0)); + return { total: [...keys.values()].reduce((a, b) => a + b, 0), keys }; } /** The engine shells out to `heygen`, which prices nothing here: under a limit those calls are refused, and diff --git a/packages/cli/src/media-use/lib/credit-limit.test.mjs b/packages/cli/src/media-use/lib/credit-limit.test.mjs index 13951117db..61abf73f70 100644 --- a/packages/cli/src/media-use/lib/credit-limit.test.mjs +++ b/packages/cli/src/media-use/lib/credit-limit.test.mjs @@ -30,7 +30,23 @@ test("spent is the last credits each job key logged", () => { const dir = project({ "heygen-jobs.jsonl": rows.map((r) => JSON.stringify(r)).join("\n") }); const spent = creditsSpent(dir); assert.equal(spent.total, 30); - assert.deepEqual([...spent.keys], ["b"]); + assert.deepEqual([...spent.keys], [["b", 30]]); +}); + +test("a malformed or null job log line is skipped", () => { + const rows = ['{"key":"a","credits":5}', "{not json", "null", '{"key":"b","credits":7}']; + assert.equal(creditsSpent(project({ "heygen-jobs.jsonl": rows.join("\n") })).total, 12); +}); + +test("a logged amount that is negative or not finite holds nothing", () => { + const rows = [ + '{"key":"a","credits":-500}', + '{"key":"b","credits":-1e999}', + '{"key":"c","credits":20}', + ]; + const spent = creditsSpent(project({ "heygen-jobs.jsonl": rows.join("\n") })); + assert.equal(spent.total, 20); + assert.deepEqual([...spent.keys.keys()], ["c"]); }); test("resolve's HeyGen voice is refused under a limit before the heygen CLI runs", async () => { diff --git a/skills-manifest.json b/skills-manifest.json index 4f9c021fb4..86ff6a860e 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "fa655ce936dab4a5", + "hash": "d96a749d23f71589", "files": 128 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/scripts/lib/credit-limit.mjs b/skills/media-use/scripts/lib/credit-limit.mjs index 02e44c6d5f..fa7b6b5ff7 100644 --- a/skills/media-use/scripts/lib/credit-limit.mjs +++ b/skills/media-use/scripts/lib/credit-limit.mjs @@ -16,18 +16,23 @@ export function creditLimit(projectDir) { return credits; } -/** Credits already committed by jobs in this project, and the job keys behind them. */ +/** Credits already committed by jobs in this project, and what each job key holds. A logged amount that is not a + * positive finite number holds nothing, so no row counts below zero. */ export function creditsSpent(projectDir) { const charged = new Map(); - for (const row of read(join(projectDir, ".hyperframes", "heygen-jobs.jsonl")).split("\n")) { - if (!row) continue; - const { key, credits } = JSON.parse(row); - if (key && typeof credits === "number") charged.set(key, credits); + for (const line of read(join(projectDir, ".hyperframes", "heygen-jobs.jsonl")).split("\n")) { + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + const { key, credits } = Object(row); + if (key && typeof credits === "number") + charged.set(key, Number.isFinite(credits) && credits > 0 ? credits : 0); } - return { - total: [...charged.values()].reduce((a, b) => a + b, 0), - keys: new Set([...charged].filter(([, c]) => c > 0).map(([k]) => k)), - }; + const keys = new Map([...charged].filter(([, c]) => c > 0)); + return { total: [...keys.values()].reduce((a, b) => a + b, 0), keys }; } /** The engine shells out to `heygen`, which prices nothing here: under a limit those calls are refused, and