Lip-sync a dubbed video with a model

Lip-sync needs a project with a finished dub and a video that shows faces. You need the scopes lipsync:write and credits:spend.

Model	Plan features	Price multiplier
`legacy`	`lipsync`, `lipsync_paid`	1
`enhanced`	`lipsync`, `lipsync_paid`, `lipsync_enhanced`	The plan’s `limits.lipsync_enhanced_cost_multiplier`

A paid run is priced on the whole video, even when the plan processes only part of it. Pricing gives the formula.

1. Estimate

const { data: estimate } = await rask.lipsync.estimateLipsync({
  path: { project_id: projectId },
  body: { model: 'enhanced' },
});

estimate = client.lipsync.estimate_lipsync(project_id, model="enhanced")

estimate, err := client.Lipsync.EstimateLipsync(ctx, projectID, api.EstimateLipsyncRequest{
	Model: api.LipsyncModelEnhanced,
})
if err != nil {
	return err
}

Estimate estimate = rask.lipsync().estimateLipsync(projectId, EstimateLipsyncRequest.builder()
        .model(LipsyncModel.ENHANCED)
        .build()).data();

$estimate = $rask->lipsync->estimateLipsync($projectId, ['model' => 'enhanced']);

curl -X POST "https://api.rask.ai/v3/projects/$PROJECT_ID/lipsync/estimate" \
  -H "Authorization: Bearer $RASK_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"model": "enhanced"}'

Besides eligible and total_minutes, details has the multiplier, the processed_seconds, whether a face check has already found a face (video_has_face, face_check_required) and the state of any earlier run (previous_run).

2. Start it

const { data: job } = await rask.lipsync.createLipsync({
  path: { project_id: projectId },
  body: { model: 'enhanced', expected_cost_minutes: 3 },
  idempotencyKey: lipsyncKey,
});

job = client.lipsync.create_lipsync(project_id, model="enhanced", expected_cost_minutes=3, idempotency_key=lipsync_key)

job, err := client.Lipsync.CreateLipsync(ctx, projectID, api.CreateLipsyncRequest{
	Model:               api.LipsyncModelEnhanced,
	ExpectedCostMinutes: 3,
}, rask.WithIdempotencyKey(lipsyncKey))
if err != nil {
	return err
}

Job job = rask.lipsync().createLipsync(
        projectId,
        CreateLipsyncRequest.builder().model(LipsyncModel.ENHANCED).expectedCostMinutes(3).build(),
        RequestOptions.builder().idempotencyKey(lipsyncKey).build()).data();

$job = $rask->lipsync->createLipsync(
    $projectId,
    ['model' => 'enhanced', 'expected_cost_minutes' => 3],
    ['idempotency_key' => $lipsyncKey],
);

LIPSYNC_KEY=$(uuidgen)
curl -X POST "https://api.rask.ai/v3/projects/$PROJECT_ID/lipsync" \
  -H "Authorization: Bearer $RASK_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: $LIPSYNC_KEY" \
  -d '{"model": "enhanced", "expected_cost_minutes": 3}'

The answer is a lipsync job: it charges, runs a face check unless one already ran, waits for a worker and renders. The job’s stage says which (face_check, waiting_for_worker, rendering).

The free run

A project has one free run. It costs 1 minute from the free lip-sync pool (lipsync_free_minutes in the balance) and processes the first 60 seconds. It needs no earlier run on the project, and free_trial_run_available in the estimate says whether it is still there. It needs the plan feature lipsync, and lipsync_enhanced for the Enhanced model, but not lipsync_paid.

const { data: job } = await rask.lipsync.createLipsync({
  path: { project_id: projectId },
  body: { model: 'legacy', free_trial_run: true, expected_cost_minutes: 1 },
  idempotencyKey: lipsyncKey,
});

job = client.lipsync.create_lipsync(
    project_id, model="legacy", free_trial_run=True, expected_cost_minutes=1, idempotency_key=lipsync_key
)

job, err := client.Lipsync.CreateLipsync(ctx, projectID, api.CreateLipsyncRequest{
	Model:               api.LipsyncModelLegacy,
	FreeTrialRun:        rask.Ptr(true),
	ExpectedCostMinutes: 1,
}, rask.WithIdempotencyKey(lipsyncKey))
if err != nil {
	return err
}

Job job = rask.lipsync().createLipsync(
        projectId,
        CreateLipsyncRequest.builder().model(LipsyncModel.LEGACY).freeTrialRun(true).expectedCostMinutes(1).build(),
        RequestOptions.builder().idempotencyKey(lipsyncKey).build()).data();

$job = $rask->lipsync->createLipsync(
    $projectId,
    ['model' => 'legacy', 'free_trial_run' => true, 'expected_cost_minutes' => 1],
    ['idempotency_key' => $lipsyncKey],
);

LIPSYNC_KEY=$(uuidgen)
curl -X POST "https://api.rask.ai/v3/projects/$PROJECT_ID/lipsync" \
  -H "Authorization: Bearer $RASK_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: $LIPSYNC_KEY" \
  -d '{"model": "legacy", "free_trial_run": true, "expected_cost_minutes": 1}'

3. Get the result

const { data: lipsyncState } = await rask.lipsync.getLipsyncState({ path: { project_id: projectId } });

lipsync_state = client.lipsync.get_lipsync_state(project_id)

lipsyncState, err := client.Lipsync.GetLipsyncState(ctx, projectID)
if err != nil {
	return err
}

LipsyncState lipsyncState = rask.lipsync().getLipsyncState(projectId).data();

$lipsyncState = $rask->lipsync->getLipsyncState($projectId);

curl "https://api.rask.ai/v3/projects/$PROJECT_ID/lipsync" \
  -H "Authorization: Bearer $RASK_API_KEY"

current_result.video.url is the video, whichever channel produced it. The same video is the output lipsync_video and, for a run started here, the job’s result.video. When the plan has no lipsync_download, download_restricted is true and the link is withheld. is_trimmed and processed_seconds say when only part was processed: the free run stops at 60 seconds, and the plan’s limits.lipsync_max_seconds can be shorter than the video.

What can go wrong

Code	When	What to do
`project_not_dubbed`	The project has no dubbed video	Dub first
`no_face_detected`	The face check found no face. As a job error the run is refunded	Stop: lip-sync needs a visible face
`lipsync_already_done`	A paid lip-sync of this dub is current, by either model. A free run does not count	Read the result. To lip-sync again, edit and redub first
`free_lipsync_already_used`	The project’s free run was used, or a paid run came first	Send `free_trial_run: false` for a paid run
`dub_in_progress`, `lipsync_in_progress`	A dub or a lip-sync runs on the project	Wait for it. `job_id` names it, or is null when that work has no job here, for example when it was started in the web app
`feature_not_in_plan`	`feature` names what is missing	Tell the user

A job that fails is refunded, whatever ended it: a failure, a timeout or a video with no face. To try again, send a new Idempotency-Key. A later dub makes the result outdated: outdated on the state is true and current_result is null, while the job’s result.video stays readable with is_outdated: true. Then you can lip-sync the new dub.

The whole program

lipsync_video.py

"""Lip-sync a dubbed project with the Enhanced model: python lipsync_video.py PROJECT_ID

Its keys are saved in lipsync-PROJECT_ID.json: run it again within 24 hours, and no paid work starts twice. Delete the file to
start anew.
"""
import sys

from rask_client import Progress, call, download, wait_for_job

project_id = sys.argv[1]
progress = Progress(f"lipsync-{project_id}.json")

estimate = call("POST", f"/projects/{project_id}/lipsync/estimate", {"model": "enhanced"})
if not estimate["eligible"]:
    sys.exit(f"Cannot start: {estimate['reasons']}")
print(f"Costs {estimate['total_minutes']} minutes, multiplier {estimate['details']['multiplier']}")

job = call(
    "POST",
    f"/projects/{project_id}/lipsync",
    {"model": "enhanced", "expected_cost_minutes": estimate["total_minutes"]},
    key=progress.key("lipsync"),
)
result = wait_for_job(job["id"])["result"]
if result["download_restricted"]:
    sys.exit("The plan withholds the download.")
download(result["video"]["url"], "lipsync.mp4")

Get started

AI agents

Recipes

Concepts

Errors

Account

Usage

Catalogs

Uploads

Media

Projects

Exports

Glossaries

Jobs

Transcript

Segments

Versions

Speakers

Voices

Voice clones

Dubbing

Lipsync

Onscreen

Lip-sync a dubbed video with a model

1. Estimate

2. Start it

The free run

3. Get the result

What can go wrong

The whole program

​1. Estimate

​2. Start it

​The free run

​3. Get the result

​What can go wrong

​The whole program

1. Estimate

2. Start it

The free run

3. Get the result

What can go wrong

The whole program