Image
ScriptName = "msLipSync"
msLipSync = {}
msLipSync.BASE_STR = 2530

-- **************************************************
-- This information is displayed in help | About scripts ...
-- **************************************************
function msLipSync:Description()
	return MOHO.Localize("/Scripts/Menu/LipSync/Description=Converts text into switch keys for a mouth layer.")
end

function msLipSync:Name()
	return "Lip Sync"
end

function msLipSync:Version()
	return "2.0"
end
function msLipSync:Creator()
	return "Mitchel Soltys"
end

-- **************************************************
-- This is the Script label in the GUI
-- **************************************************

function msLipSync:UILabel()
	return(MOHO.Localize("/Scripts/Menu/LipSync/LayerLipSync=LipSync..."))
end

-- **************************************************
-- Recurring values
-- **************************************************
msLipSync.stepSize = 1
msLipSync.startFrame = 1
msLipSync.endFrame = 50
msLipSync.text = ""
msLipSync.switch = nil
msLipSync.skel = nil
msLipSync.cancel = false
msLipSync.phonemeToBonesMap = {}
msLipSync.phonetic = true
msLipSync.consonantFrames = 1
msLipSync.firstChecked = 0

-- **************************************************
-- LipSync dialog
-- **************************************************

local msLipSyncDialog = {}

function msLipSyncDialog:new(moho)
	msHelper:Debug("in sync dialog ")


	local dialog, layout = msDialog:SimpleDialog("Lip Sync", msLipSyncDialog)

	dialog.moho = moho
    msLipSync.myPhonemes = msPhonemes.new()
	-- msLipSync.myPhonemes.BuildPhonemeMap(moho:AppDir().."/scripts/utility/lipSync.txt")
	msLipSync.myPhonemes.BuildPhonemeMap(moho:UserAppDir().."/scripts/utility/lipSync.txt")
	--myPhonemes.dump(myPhonemes.boneMaps)
	msHelper:Debug("after buildPhonememap  ")

    dialog.isBoneLayer = false
	if (msLipSync.moho.layer:LayerType() == MOHO.LT_BONE) then
		dialog.syncMenu = LM.GUI.Menu(msDialog:Localize("SyncStyle",
				"LipSync Style:"))
		dialog.isBoneLayer = true
	  --msPhonemes:dump(msPhonemes.boneMaps)
		local numMaps = 0
		for k,v in pairs(msLipSync.myPhonemes.boneMaps) do
			if numMaps == 0 then msLipSync.myPhonemes.setBoneMap(k) end
			dialog.syncMenu:AddItem(k, 0, MOHO.MSG_BASE + numMaps)
			numMaps = numMaps + 1
		end
		msDialog:MakePopup(dialog.syncMenu)
     end		
	
    --print("first checked label " .. dialog.syncMenu:FirstCheckedLabel())
	layout:PushH(LM.GUI.ALIGN_CENTER)
		-- add labels
		layout:PushV()
			msDialog:AddText("Start Frame:")
			msDialog:AddText("End Frame:")
			msDialog:AddText("Text String:")
			msDialog:AddText("Consonant Frames:")
	
	-- Should the rest mouth be put at the end of the audio section
	dialog.phonetic = msDialog:Control(LM.GUI.CheckBox, "Phonetic","Phonetic spelling")
	dialog.debug = msDialog:Control(LM.GUI.CheckBox, "Debug","Debug")

		layout:Pop()
		-- add controls to the right
		layout:PushV()
			dialog.startFrame = msDialog:AddTextControl(0, "1.0000", 0, LM.GUI.FIELD_FLOAT)
			dialog.endFrame = msDialog:AddTextControl(0, "100.0000", 0, LM.GUI.FIELD_FLOAT)
			dialog.text = msDialog:AddTextControl(0,"What are you saying", 0, LM.GUI.FIELD_TEXT)
			dialog.consonantFrames = msDialog:AddTextControl(0,"1.0000", 0, LM.GUI.FIELD_INT)
		layout:Pop()
	layout:Pop()
	
	return dialog
end

-- **************************************************
-- Set dialog values
-- **************************************************
function msLipSyncDialog:UpdateWidgets()
	self.startFrame:SetValue(msLipSync.startFrame)
	self.endFrame:SetValue(msLipSync.endFrame)
	self.consonantFrames:SetValue(msLipSync.consonantFrames)
	self.text:SetValue(msLipSync.text)
	self.phonetic:SetValue(msLipSync.phonetic)
	self.debug:SetValue(msHelper.debug)
	if self.isBoneLayer then 
		self.syncMenu:SetChecked(MOHO.MSG_BASE + msLipSync.firstChecked, true)
		msLipSync.firstChecked = self.syncMenu:FirstChecked()
	end
end

-- **************************************************
-- Validate user choices
-- **************************************************

-- **************************************************
-- Set values from dialog
-- **************************************************
function msLipSyncDialog:OnOK()
	msHelper:Debug("in sync OnOK  ")

	msLipSync.startFrame =	self.startFrame:FloatValue()
    msLipSync.endFrame = self.endFrame:FloatValue()
    msLipSync.consonantFrames = self.consonantFrames:IntValue()
    msLipSync.text = self.text:Value()
    msLipSync.phonetic = self.phonetic:Value()
	msHelper.debug = self.debug:Value()
	if self.isBoneLayer then 
	    msHelper:Debug("this is a Bonelayer ")
		msHelper:Debug("in lipsyncdialog:OnOK - bone map  " .. self.syncMenu:FirstCheckedLabel())
		msLipSync.boneMapName = self.syncMenu:FirstCheckedLabel()
		msLipSync.myPhonemes.setBoneMap(msLipSync.boneMapName)
		msLipSync.firstChecked = self.syncMenu:FirstChecked()
	end
	msHelper:Debug("leaving sync OnOK  ")

	--msLipSync.boneMap = msPhonemes.boneMaps[1]
	--msPhonemes.dump(msLipSync.boneMap)
end

-- **************************************************
-- The guts of this script
-- **************************************************
function msLipSync:DeleteKeys()
	for frame = self.startFrame, self.endFrame do
		-- self.switch:DeleteKey(frame)
	end
end

function msLipSync:CalculateStepSize(phonemeList)
  local numVowels, numConsonants = self.myPhonemes.countPhonemes(phonemeList)
	self.stepSize = (self.endFrame - self.startFrame - numConsonants)/numVowels
  if self.stepSize < 1 then self.stepSize = 1 end
  msHelper:Debug("Frames " .. self.startFrame .. " " .. self.endFrame)
  msHelper:Debug("Letters v, c, ss " .. numVowels .. " " .. numConsonants .. " " .. self.stepSize)

end

function msLipSync:dump(table)
    for k, v in pairs(table) do
        if (type(v) == "table") then
            self:dump(v)
        else
            print("key " .. k .. " value " .. v)
        end
    end
end


function msLipSync:SetMouthValues(phonemeList,type)
	local frame = self.startFrame + self.frameAdjust
	local numBones = 0
	local lastMouth = 0
	if type ~= "switch" then
		numBones = self.myPhonemes.numBones()
		msHelper:Debug("numBones " .. numBones)
	end
    msHelper:Debug("in setMouthValues")
	msHelper:Debug("type is " .. type)
	for k,v in ipairs(phonemeList) do
		local mouth = v[1]
		msHelper:Debug("mouth is  " .. mouth)
		msHelper:Debug("frame " .. frame)
		if (mouth ~= lastMouth) then
			if(type == "switch") then
				self.switch:SetValue(math.floor(frame), mouth)
			else
				for i = 1, numBones, 1 do
					msHelper:Debug("bone name " .. self.myPhonemes.boneName(i))
					msHelper:Debug("bone angle " .. self.myPhonemes.boneAngle(i,mouth))
					local bone = self.skel:BoneByName(self.myPhonemes.boneName(i))
					if (bone == nil) then 
						print("The bone '" .. self.myPhonemes.boneName(i) .. "' is not found.")
						print("Make sure " .. msLipSync.boneMapName .. " is the correct bone map to use.")
						return
					end
					bone.fAnimAngle:SetValue(frame,self.myPhonemes.boneAngleRad(i, mouth))
				end
			end
			lastMouth = mouth
		end
		if v[2] == "c" then 
			frame = frame + self.consonantFrames
		else
		  frame = frame + self.stepSize
		end
	end
end

function msLipSync:IsEnabled(moho)
	if ((moho.layer:LayerType() ~= MOHO.LT_SWITCH) and (moho.layer:LayerType() ~= MOHO.LT_BONE)) then
		return false
	end
	return true
end

function msLipSync:OnMouseDown(moho, mouseEvent)
	self.startFrame = moho.layer:CurFrame()
end


function msLipSync:OnMouseUp(moho, mouseEvent)
	-- if (mouseEvent.shiftKey) then
		self.endFrame = moho.layer:CurFrame()
		self:DoLipSync(moho)
	-- else
		-- self.startFrame = moho.layer:CurFrame()
	-- end
end

function msLipSync:OnMouseMoved(moho, mouseEvent)

	mouseGenValue=mouseEvent.pt.x-mouseEvent.startPt.x
	newFrame=self.startFrame+math.floor(mouseGenValue/10)
	moho:SetCurFrame(newFrame)

end

function msLipSync:DoLipSync(moho)
	self.moho = moho
	msDialog.cancelled = false
	msHelper:Debug("in run before dialog  ")
	if (msDialog:Display(moho, msLipSyncDialog) == LM.GUI.MSG_CANCEL) then
		msHelper:Debug("msDialog is cancelled" )
		return
	end	
	
	msHelper:Debug("in run after dialog  ")
	self.frameAdjust = moho.layer:TotalTimingOffset();


	moho.document:PrepUndo(moho.layer)
	moho.document:SetDirty()

	
	local phonemeList = {}
	msHelper:Debug("phrase to speak " .. self.text)
	self.myPhonemes.buildPhonemeListFromPhrase(self.text, phonemeList, self.phonetic)
	self:CalculateStepSize(phonemeList)
    msHelper:Debug("after size calculation")
	
	if (moho.layer:LayerType() == MOHO.LT_SWITCH) then 
		local switchLayer = moho:LayerAsSwitch(moho.layer)
		self.switch = switchLayer:SwitchValues()
		self:SetMouthValues(phonemeList,"switch")
	else 
		self.skel = moho:Skeleton()
		if (self.skel == nil) then
			msHelper:Debug("self.skel is nil")
			return
		end
		self:SetMouthValues(phonemeList,"bone")
	end
	moho.layer:UpdateCurFrame(true)
	moho:UpdateUI()
end

Icon
msLipSync
Listed

Script type: Tool

Uploaded: Jan 29 2022, 08:48

Allows animators to easily add lip sync to characters.

This script allows you to easily do lip sync for both, characters using switch layers for mouth shapes, and characters using bones to control the mouth (bone names need to be a single word). The script will work with Anime Studio 11 up to and including the latest version of Moho. I haven't tested recently with versions older than 11, but it might work there as well.


Installation
=====
The following files should be downloaded from the script. Place them in the script directories as shown. Then restart moho.

scripts\tool\msLipSync.lua - the main code for the lipSync script
scripts\tool\msLipSync.png - the icon displayed for the tool
scripts\utility\msDialog.lua - dialog utility routines
scripts\utility\msHelper.lua - debugging routines
scripts\utility\msPhonemes.lua - routines for breaking text into phonemes


Apparently, text files cannot be uploaded here, so the following lines will need to be copied into a file called, "lipSync.txt" and placed in the scripts\utilty directory. It is a bone map (described below) for characters that have mouths controlled by bones. It needs to be in the utility directory, even if you use only switch layers, because the lip sync script expects it to be there. You can change the bone angles to your liking. The discussion below explains how.

Scarlett 2
Open/Close Squash/Stretch
AI 304 349
E 278 349
L 267 353
etc 258 349
FV 233 349
MBP 203 349
rest 223 368
O 306 300
U 306 325
WQ 193 266
Dragon 1
B70
AI 280
E 285
L 285
etc 290
FV 295
MBP 298
rest 298
O 285
U 290
WQ 295
Gorilla 1
Menace
AI 156
E 161
L 167
etc 169
FV 169
MBP 176
rest 187
O 160
U 160
WQ 160
Bigsby 1
Open/Close
AI 245
E 216
L 216
etc 198
FV 183
MBP 180
rest 180
O 203
U 197
WQ 188
Gruille 1
Mouth
AI 371
E 380
L 380
etc 390
FV 395
MBP 403
rest 403
O 380
U 390
WQ 395
end

Functionality
=============
The script allows the user to specify a phrase in a dialog box, as well
as a starting and ending frame. It will then create keyframes for the
character, matching the phrase entered by the user.

Usage
=====
This video describes the script
https://www.youtube.com/watch?v=bb8dVbtTRrI

The script is named Lip Sync. In order to select the script you
must select either a switch layer, that controls the standard mouth
shapes, or a bone layer that controls mouth movements. If you are
not on one of those layers the script will not be enabled.

After you have selected a mouth switch layer or bone layer, the Lip Sync
tool will be enabled. Next, move the timeline marker to the starting point
of the audio you want to lip sync. Now, move the cursor to the workspace
and drag until the timeline marker is moved to the place where you want
the lip sync to end. (Note: You need to drag in the workspace area, because
the scripting does not allow you to drag within the timeline ares.)

When you have finished dragging the mouse, let the mouse button up and
a dialog will come up. The start frame and end frame will automatically
be populated by your mouse drag. Next enter the text for the audio.
The dialog allows you to select Phonetic spelling and Debug.

When you select ok the script will add key frames to the layer you
have selected. It will not delete existing key frames.

Tips for Lip Sync
=================
- Use phonetic spelling for greater control.
- Repeat vowels to have them sound for a longer time.
- For pauses, just lip sync each section separately.
- Use phonetic spelling and captial letters for vowels that you want long pronunciation.
- Use a hyphen whenever you want to have the "rest" mouth shape.
- Increase the consonant frame count if you consonants seem to be held too short.
- Tweak the result after generating key frames, to meet your needs.

Intention of the Script
=======================
While the script does an excellent job spacing out phonemes, the
intention is not specifically to get a perfect match. Rather the
intention is to get the proper sequence of phonemes in the correct
order and close to the correct location. You can then tweak the
speech by dragging key frames as normal.

Consonants, Vowels and Held Tones
=================================
The script recognizes that, in general, many consonants cannot be held.
For example, in the word, "cat", "c" and "t" cannot really be held a
long time. By default, those kinds of letters are only given one
frame. The script does allow you to specify the length of the
consonant sounds.

Vowels, on the other hand can easily be held for a long time. As
a result the script will assign all the short consonants the specified
short time, subtract them from the total number of frames and then divide the
result by the number of vowels. This creates a phoneme spacing that
matches the speech in most cases.

If your speech has certain sounds
that are held inordinantly long you simply need to list those sounds
multiple times. For example "the faaat cat" would hold the "a" sound,
in "fat", three times longer than the other vowels in the phrase. The
script also recognizes that certain consonants, like "n" and "m" can
be held, so they act like vowels.

Non-letters
===========
Punctuation and numbers should not cause the script to crash, but the
symbols will be unrecognized and will, in most cases, result in an
"etc" phoneme.

Phonetic or not
===============
The script does a fair job at converting basic English phrases into the
appropriate phonemes, but using phonetic spelling matches the phonemes
exactly to what you want. You can even use it to match foreign language
by and non-language, just by selecting whatever phonetic symbol you
want. The exact phonetic mapping is given at the end of this file.
For vowels, capital letters produce long vowel sounds. Lower cases
vowels result in short pronunciation.

Debug
=====
You can select Debug if something goes wrong in the lip sync. With
Debug selected, statements will be printed to hopefully help fix the
issue. Note, since I do not earn my income from the script I cannot
support problems that might occur, but hopefully the Debug option can
help if you encounter a problem.

Switch Layers
=============
If you are using switch layers for lip sync, it needs to have the
standard mouth shapes. It can have other shapes, and they can be in
any order, but it must have the shapes AI, E, L, FV, etc, MBP, O, U,
WQ, rest. Note that capitalization does matter.

Bone Layers
===========
If you are using bones to control your mouth, the dialog will show a
selection box where you will choose the appropriate bone map to use
with your character. You can modify the file Utility\lipSync.txt to
lip sync any bones controlling you character's speech.

Bone Maps
=========
A bone map tells the script what angles to set for which bones in order
to produce the mouth look you want for any of the standard mouth
phonemes. You can have any number of bones controlling your mouth. Bone
maps that come with this script include Scarlett (used with Scarlett Riggs),
Bigsby, Dragon, Gorilla, and Gruille.

Creating Your Own Bone Maps
===========================
Creating your own bone maps is easy, but it follows very specific
rules. Failure to follow the rules will cause the script to break.
The easiest way to create a bone map is to copy an existing one and
modify it. You don't add a new file, you just insert another map into
the same file. The format of the map is

mapName numberOfBones
boneName1 boneName2 ... boneNameN
phoneme boneAngle1 boneAngle2 ... boneAngleN

Rules of the Bone Maps
======================
- The last line in the file must be the word "end"
- mapName and boneNames must be a single word
- the mapName can be anything, but the boneNames must match real bones
in the character
- a forward slash, /,  is allowed in names, but other characters might cause the script to
break
- capitalization matters for bone names
- you need to include bone angles for each phoneme
  - AI E L etc FV MBP rest O U WQ 
- capitalization matters for phoneme names
- bone angles need to be in degrees

A Word of Advice About Creating Your Own Maps
=============================================
Before you create your own bone map, study the ones that come with the
script. Next create a very simple map with a single bone. If a bone map
doesn't seem to work, try to simplify it and test with a very simple
phrase.

Phonetic Spelling
=================
The following list shows the phonetic letter and the phoneme that will
be created

    a = AI
    b = MBP
    c = etc
    d = etc
    e = E
    f = FV
    g = etc
    h = etc
    i = E
    j = etc
    k = etc
    l = L
    m = MBP
    n = etc
    o = O
    p = MBP
    q = WQ
    r = etc
    s = etc
    t = etc
    u = U
    v = FV
    w = WQ
    x = etc
    y = etc
    z = etc
    A = AI
    E = E
    I = AI
    U = U
    O = O
    R = O
  - = rest

Notice that in many cases the long and short vowel have the same mouth
phoneme. That just makes it easier to specify the spelling. Notice
also, that you may specify "-" so signify the "rest" mouth position.
This script, and all other scripts on this site are distributed as free software under the GNU General Public License 3.0 or later.
Downloads count: 1713