From ae033b21359489ef8bb5ce3d17e60b70e43c340c Mon Sep 17 00:00:00 2001 From: K-bai Date: Fri, 6 Dec 2024 14:03:33 +0800 Subject: [PATCH] feat: play multiple motions in parallel --- README.md | 12 ++ README.zh.md | 11 ++ src/Live2DModel.ts | 27 +++ src/cubism-common/InternalModel.ts | 10 + src/cubism-common/ParallelMotionManager.ts | 206 ++++++++++++++++++++ src/cubism2/Cubism2InternalModel.ts | 13 +- src/cubism2/Cubism2ParallelMotionManager.ts | 50 +++++ src/cubism4/Cubism4InternalModel.ts | 13 +- src/cubism4/Cubism4ParallelMotionManager.ts | 62 ++++++ 9 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 src/cubism-common/ParallelMotionManager.ts create mode 100644 src/cubism2/Cubism2ParallelMotionManager.ts create mode 100644 src/cubism4/Cubism4ParallelMotionManager.ts diff --git a/README.md b/README.md index 1e5c3940..1efa50bd 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,18 @@ window.PIXI = PIXI; })(); ``` +### play multiple motions in parallel + +```ts +model.parallelMotion([ + {group: motion_group1, index: motion_index1, priority: MotionPriority.NORMAL}, + {group: motion_group2, index: motion_index2, priority: MotionPriority.NORMAL}, +]); +``` + +If you need to synchronize the playback of expressions and sounds, please use`model.motion`/`model.speak` to play one of the motions, and use `model.parallelMotion` to play the remaining motions. Each item in the motion list has independent priority control based on its index, consistent with the priority logic of `model.motion`. + + ## Package importing When importing Pixi packages on-demand, you may need to manually register some plugins to enable optional features. diff --git a/README.zh.md b/README.zh.md index 853d9666..66e602da 100644 --- a/README.zh.md +++ b/README.zh.md @@ -134,6 +134,17 @@ window.PIXI = PIXI; })(); ``` +### 多动作同步播放 + +```ts +model.parallelMotion([ + {group: motion_group1, index: motion_index1, priority: MotionPriority.NORMAL}, + {group: motion_group2, index: motion_index2, priority: MotionPriority.NORMAL}, +]); +``` + +若需要同步播放表情、声音等请使用`model.motion`/`model.speak`播放其中一个动作，其余动作用`model.parallelMotion`播放。列表中按照index每一项都有独立的优先级控制，和`model.motion`逻辑一致。 + ## 包导入 当按需导入 Pixi 的包时，需要手动注册相应的组件来启用可选功能 diff --git a/src/Live2DModel.ts b/src/Live2DModel.ts index 3df8070b..76810af2 100644 --- a/src/Live2DModel.ts +++ b/src/Live2DModel.ts @@ -178,6 +178,33 @@ export class Live2DModel extends Conta } return Promise.resolve(false); } + + /** + * Shorthand to start multiple motions in parallel. + * @param motionList - The motion list: { + * group: The motion group, + * index: Index in the motion group, + * priority - The priority to be applied. (0: No priority, 1: IDLE, 2:NORMAL, 3:FORCE) (default: 2) + * }[] + * @return Promise that resolves with a list, indicates the motion is successfully started, with false otherwise. + */ + async parallelMotion( + motionList: { + group: string, + index: number, + priority?: MotionPriority, + }[] + ): Promise { + this.internalModel.extendParallelMotionManager(motionList.length); + const result = motionList.map((m, idx) => ( + this.internalModel.parallelMotionManager[idx]?.startMotion(m.group, m.index, m.priority) + )); + let flags = []; + for (let r of result) { + flags.push(await r!); + } + return flags; + } /** * Updates the focus position. This will not cause the model to immediately look at the position, diff --git a/src/cubism-common/InternalModel.ts b/src/cubism-common/InternalModel.ts index 126b9b36..05683056 100644 --- a/src/cubism-common/InternalModel.ts +++ b/src/cubism-common/InternalModel.ts @@ -1,6 +1,7 @@ import { FocusController } from "@/cubism-common/FocusController"; import type { ModelSettings } from "@/cubism-common/ModelSettings"; import type { MotionManager, MotionManagerOptions } from "@/cubism-common/MotionManager"; +import type { ParallelMotionManager } from "@/cubism-common/ParallelMotionManager"; import { LOGICAL_HEIGHT, LOGICAL_WIDTH } from "@/cubism-common/constants"; import { Matrix, utils } from "@pixi/core"; import type { Mutable } from "../types/helpers"; @@ -56,6 +57,7 @@ export abstract class InternalModel extends utils.EventEmitter { focusController = new FocusController(); abstract motionManager: MotionManager; + abstract parallelMotionManager: ParallelMotionManager[]; pose?: any; physics?: any; @@ -270,6 +272,8 @@ export abstract class InternalModel extends utils.EventEmitter { this.motionManager.destroy(); (this as Partial).motionManager = undefined; + this.parallelMotionManager.forEach(m => m.destroy()); + this.parallelMotionManager = []; } /** @@ -326,4 +330,10 @@ export abstract class InternalModel extends utils.EventEmitter { * Draws the model. */ abstract draw(gl: WebGLRenderingContext): void; + + /** + * Add parallel motion manager. + * @param managerCount - Count of parallel motion managers. + */ + abstract extendParallelMotionManager(managerCount: number): void; } diff --git a/src/cubism-common/ParallelMotionManager.ts b/src/cubism-common/ParallelMotionManager.ts new file mode 100644 index 00000000..1a4ec280 --- /dev/null +++ b/src/cubism-common/ParallelMotionManager.ts @@ -0,0 +1,206 @@ +import type { MotionManager } from "@/cubism-common/MotionManager"; +import type { ModelSettings } from "@/cubism-common/ModelSettings"; +import { MotionPriority, MotionState } from "@/cubism-common/MotionState"; +import { logger } from "@/utils"; +import { utils } from "@pixi/core"; +import type { Mutable } from "../types/helpers"; + + + + +/** + * Handles the motion playback. + * @emits {@link MotionManagerEvents} + */ +export abstract class ParallelMotionManager extends utils.EventEmitter { + /** + * Tag for logging. + */ + tag: string; + + manager: MotionManager; + + + /** + * The ModelSettings reference. + */ + readonly settings: ModelSettings; + + + /** + * Maintains the state of this MotionManager. + */ + state = new MotionState(); + + + /** + * Flags there's a motion playing. + */ + playing = false; + + /** + * Flags the instances has been destroyed. + */ + destroyed = false; + + protected constructor(settings: ModelSettings, manager: MotionManager) { + super(); + this.settings = settings; + this.tag = `ParallelMotionManager(${settings.name})`; + this.state.tag = this.tag; + this.manager = manager; + } + + /** + * Starts a motion as given priority. + * @param group - The motion group. + * @param index - Index in the motion group. + * @param priority - The priority to be applied. default: 2 (NORMAL) + * ### OPTIONAL: {name: value, ...} + * @param sound - The audio url to file or base64 content + * @param volume - Volume of the sound (0-1) + * @param expression - In case you want to mix up a expression while playing sound (bind with Model.expression()) + * @param resetExpression - Reset expression before and after playing sound (default: true) + * @param crossOrigin - Cross origin setting. + * @return Promise that resolves with true if the motion is successfully started, with false otherwise. + */ + async startMotion( + group: string, + index: number, + priority: MotionPriority = MotionPriority.NORMAL, + ): Promise { + if (!this.state.reserve(group, index, priority)) { + return false; + } + + + const definition = this.manager.definitions[group]?.[index]; + if (!definition) { + return false; + } + + const motion = await this.manager.loadMotion(group, index); + + + if (!this.state.start(motion, group, index, priority)) { + return false; + } + logger.log(this.tag, "Start motion:", this.getMotionName(definition)); + + this.emit("motionStart", group, index, undefined); + + this.playing = true; + + this._startMotion(motion!); + + return true; + } + + /** + * Starts a random Motion as given priority. + * @param group - The motion group. + * @param priority - The priority to be applied. (default: 1 `IDLE`) + * ### OPTIONAL: {name: value, ...} + * @param sound - The wav url file or base64 content+ + * @param volume - Volume of the sound (0-1) (default: 1) + * @param expression - In case you want to mix up a expression while playing sound (name/index) + * @param resetExpression - Reset expression before and after playing sound (default: true) + * @return Promise that resolves with true if the motion is successfully started, with false otherwise. + */ + async startRandomMotion( + group: string, + priority?: MotionPriority + ): Promise { + const groupDefs = this.manager.definitions[group]; + + if (groupDefs?.length) { + const availableIndices: number[] = []; + + for (let i = 0; i < groupDefs!.length; i++) { + if (this.manager.motionGroups[group]![i] !== null && !this.state.isActive(group, i)) { + availableIndices.push(i); + } + } + + if (availableIndices.length) { + const index = + availableIndices[Math.floor(Math.random() * availableIndices.length)]!; + + return this.startMotion(group, index, priority); + } + } + + return false; + } + + /** + * Stops all playing motions as well as the sound. + */ + stopAllMotions(): void { + this._stopAllMotions(); + + this.state.reset(); + + } + + /** + * Updates parameters of the core model. + * @param model - The core model. + * @param now - Current time in milliseconds. + * @return True if the parameters have been actually updated. + */ + update(model: object, now: DOMHighResTimeStamp): boolean { + if (this.isFinished()) { + if (this.playing) { + this.playing = false; + this.emit("motionFinish"); + } + + this.state.complete(); + } + return this.updateParameters(model, now); + } + + + /** + * Destroys the instance. + * @emits {@link MotionManagerEvents.destroy} + */ + destroy() { + this.destroyed = true; + this.emit("destroy"); + + this.stopAllMotions(); + + const self = this as Mutable>; + } + + /** + * Checks if the motion playback has finished. + */ + abstract isFinished(): boolean; + + /** + * Retrieves the motion's name by its definition. + * @return The motion's name. + */ + protected abstract getMotionName(definition: MotionSpec): string; + + /** + * Starts the Motion. + */ + protected abstract _startMotion(motion: Motion, onFinish?: (motion: Motion) => void): number; + + /** + * Stops all playing motions. + */ + protected abstract _stopAllMotions(): void; + + /** + * Updates parameters of the core model. + * @param model - The core model. + * @param now - Current time in milliseconds. + * @return True if the parameters have been actually updated. + */ + protected abstract updateParameters(model: object, now: DOMHighResTimeStamp): boolean; +} diff --git a/src/cubism2/Cubism2InternalModel.ts b/src/cubism2/Cubism2InternalModel.ts index 9b1dca9f..570466af 100644 --- a/src/cubism2/Cubism2InternalModel.ts +++ b/src/cubism2/Cubism2InternalModel.ts @@ -4,6 +4,7 @@ import { InternalModel } from "@/cubism-common/InternalModel"; import { logger } from "../utils"; import type { Cubism2ModelSettings } from "./Cubism2ModelSettings"; import { Cubism2MotionManager } from "./Cubism2MotionManager"; +import { Cubism2ParallelMotionManager } from "./Cubism2ParallelMotionManager"; import { Live2DEyeBlink } from "./Live2DEyeBlink"; import type { Live2DPhysics } from "./Live2DPhysics"; import type { Live2DPose } from "./Live2DPose"; @@ -21,6 +22,7 @@ export class Cubism2InternalModel extends InternalModel { coreModel: Live2DModelWebGL; motionManager: Cubism2MotionManager; + parallelMotionManager: Cubism2ParallelMotionManager[]; eyeBlink?: Live2DEyeBlink; @@ -61,6 +63,7 @@ export class Cubism2InternalModel extends InternalModel { this.coreModel = coreModel; this.settings = settings; this.motionManager = new Cubism2MotionManager(settings, options); + this.parallelMotionManager = []; this.eyeBlink = new Live2DEyeBlink(coreModel); this.eyeballXParamIndex = coreModel.getParamIndex("PARAM_EYE_BALL_X"); @@ -226,7 +229,9 @@ export class Cubism2InternalModel extends InternalModel { this.emit("beforeMotionUpdate"); - const motionUpdated = this.motionManager.update(this.coreModel, now); + const motionUpdated0 = this.motionManager.update(model, now); + const parallelMotionUpdated = this.parallelMotionManager.map(m => m.update(model, now)); + const motionUpdated = motionUpdated0 || parallelMotionUpdated.reduce((prev, curr) => prev || curr, false); this.emit("afterMotionUpdate"); @@ -297,6 +302,12 @@ export class Cubism2InternalModel extends InternalModel { this.hasDrawn = true; this.disableCulling = disableCulling; } + + extendParallelMotionManager(managerCount: number) { + while (this.parallelMotionManager.length < managerCount) { + this.parallelMotionManager.push(new Cubism2ParallelMotionManager(this.settings, this.motionManager)) + } + } destroy() { super.destroy(); diff --git a/src/cubism2/Cubism2ParallelMotionManager.ts b/src/cubism2/Cubism2ParallelMotionManager.ts new file mode 100644 index 00000000..e5df55bf --- /dev/null +++ b/src/cubism2/Cubism2ParallelMotionManager.ts @@ -0,0 +1,50 @@ +import { config } from "@/config"; +import type { MotionManager } from "@/cubism-common/MotionManager"; +import { ParallelMotionManager } from "@/cubism-common/ParallelMotionManager"; +import { Cubism2ExpressionManager } from "@/cubism2/Cubism2ExpressionManager"; +import type { Cubism2ModelSettings } from "@/cubism2/Cubism2ModelSettings"; +import type { Cubism2Spec } from "../types/Cubism2Spec"; +import type { Mutable } from "../types/helpers"; +import "./patch-motion"; + +export class Cubism2ParallelMotionManager extends ParallelMotionManager { + readonly queueManager = new MotionQueueManager(); + + constructor(settings: Cubism2ModelSettings, manager: MotionManager) { + super(settings, manager); + } + + + isFinished(): boolean { + return this.queueManager.isFinished(); + } + + protected getMotionName(definition: Cubism2Spec.Motion): string { + return definition.file; + } + + protected _startMotion( + motion: Live2DMotion, + onFinish?: (motion: Live2DMotion) => void, + ): number { + motion.onFinishHandler = onFinish; + + this.queueManager.stopAllMotions(); + + return this.queueManager.startMotion(motion); + } + + protected _stopAllMotions(): void { + this.queueManager.stopAllMotions(); + } + + protected updateParameters(model: Live2DModelWebGL, now: DOMHighResTimeStamp): boolean { + return this.queueManager.updateParam(model); + } + + destroy() { + super.destroy(); + + (this as Partial>).queueManager = undefined; + } +} diff --git a/src/cubism4/Cubism4InternalModel.ts b/src/cubism4/Cubism4InternalModel.ts index 35b4581a..74e11bef 100644 --- a/src/cubism4/Cubism4InternalModel.ts +++ b/src/cubism4/Cubism4InternalModel.ts @@ -3,6 +3,7 @@ import type { CommonHitArea, CommonLayout } from "@/cubism-common/InternalModel" import { InternalModel } from "@/cubism-common/InternalModel"; import type { Cubism4ModelSettings } from "@/cubism4/Cubism4ModelSettings"; import { Cubism4MotionManager } from "@/cubism4/Cubism4MotionManager"; +import { Cubism4ParallelMotionManager } from "@/cubism4/Cubism4ParallelMotionManager"; import { ParamAngleX, ParamAngleY, @@ -29,6 +30,7 @@ export class Cubism4InternalModel extends InternalModel { settings: Cubism4ModelSettings; coreModel: CubismModel; motionManager: Cubism4MotionManager; + parallelMotionManager: Cubism4ParallelMotionManager[]; lipSync = true; @@ -73,6 +75,7 @@ export class Cubism4InternalModel extends InternalModel { this.coreModel = coreModel; this.settings = settings; this.motionManager = new Cubism4MotionManager(settings, options); + this.parallelMotionManager = []; this.init(); } @@ -201,7 +204,9 @@ export class Cubism4InternalModel extends InternalModel { this.emit("beforeMotionUpdate"); - const motionUpdated = this.motionManager.update(this.coreModel, now); + const motionUpdated0 = this.motionManager.update(model, now); + const parallelMotionUpdated = this.parallelMotionManager.map(m => m.update(model, now)); + const motionUpdated = motionUpdated0 || parallelMotionUpdated.reduce((prev, curr) => prev || curr, false); this.emit("afterMotionUpdate"); @@ -268,6 +273,12 @@ export class Cubism4InternalModel extends InternalModel { this.renderer.setRenderState(gl.getParameter(gl.FRAMEBUFFER_BINDING), this.viewport); this.renderer.drawModel(); } + + extendParallelMotionManager(managerCount: number) { + while (this.parallelMotionManager.length < managerCount) { + this.parallelMotionManager.push(new Cubism4ParallelMotionManager(this.settings, this.motionManager)) + } + } destroy() { super.destroy(); diff --git a/src/cubism4/Cubism4ParallelMotionManager.ts b/src/cubism4/Cubism4ParallelMotionManager.ts new file mode 100644 index 00000000..68d2e91b --- /dev/null +++ b/src/cubism4/Cubism4ParallelMotionManager.ts @@ -0,0 +1,62 @@ +import { ParallelMotionManager } from "@/cubism-common/ParallelMotionManager"; +import type { Cubism4ModelSettings } from "@/cubism4/Cubism4ModelSettings"; +import type { CubismSpec } from "@cubism/CubismSpec"; +import type { CubismModel } from "@cubism/model/cubismmodel"; +import type { ACubismMotion } from "@cubism/motion/acubismmotion"; +import { CubismMotion } from "@cubism/motion/cubismmotion"; +import { CubismMotionQueueManager } from "@cubism/motion/cubismmotionqueuemanager"; +import type { Mutable } from "../types/helpers"; +import type { MotionManager } from "@/cubism-common/MotionManager"; + +import { logger } from "@/utils"; + +export class Cubism4ParallelMotionManager extends ParallelMotionManager { + readonly queueManager = new CubismMotionQueueManager(); + + declare readonly settings: Cubism4ModelSettings; + + constructor(settings: Cubism4ModelSettings, manager: MotionManager) { + super(settings, manager); + + this.init(); + } + + protected init() { + this.queueManager.setEventCallback((caller, eventValue, customData) => { + this.emit("motion:" + eventValue); + }); + } + + isFinished(): boolean { + return this.queueManager.isFinished(); + } + + protected _startMotion( + motion: CubismMotion, + onFinish?: (motion: CubismMotion) => void, + ): number { + motion.setFinishedMotionHandler(onFinish as (motion: ACubismMotion) => void); + + this.queueManager.stopAllMotions(); + return this.queueManager.startMotion(motion, false, performance.now()); + } + + protected _stopAllMotions(): void { + this.queueManager.stopAllMotions(); + } + + protected updateParameters(model: CubismModel, now: DOMHighResTimeStamp): boolean { + return this.queueManager.doUpdateMotion(model, now); + } + + protected getMotionName(definition: CubismSpec.Motion): string { + return definition.File; + } + + destroy() { + super.destroy(); + + this.queueManager.release(); + (this as Partial>).queueManager = undefined; + } +}