1 Commits

Author SHA1 Message Date
Ada
d44ccc3ace feat(flow): add interactive AI puppet to FlowScreen
- Added puppet component modules: PuppetView, FlowPuppetSlot, and type definitions

- The puppet supports actions such as idle/smile/jump/shake/think, with a default smile.

- FlowScreen integrates puppet slots; it automatically uses the "think" function when sending messages and allows for interactive actions like Smile/Jump/Shake.

- The code is independent of the existing chat logic and does not affect existing functionality.
2026-02-04 16:57:28 -08:00
7 changed files with 633 additions and 0 deletions

157
package-lock.json generated
View File

@@ -36,6 +36,7 @@
"react-native-reanimated": "~3.16.1",
"react-native-safe-area-context": "4.12.0",
"react-native-screens": "~4.4.0",
"react-native-svg": "^15.15.2",
"react-native-view-shot": "^3.8.0",
"react-native-web": "~0.19.13",
"readable-stream": "^4.7.0",
@@ -4705,6 +4706,12 @@
"@noble/hashes": "^1.2.0"
}
},
"node_modules/boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
"license": "ISC"
},
"node_modules/bplist-creator": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/bplist-creator/-/bplist-creator-0.0.7.tgz",
@@ -5441,6 +5448,56 @@
"utrie": "^1.0.2"
}
},
"node_modules/css-select": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
"integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
"license": "BSD-2-Clause",
"dependencies": {
"boolbase": "^1.0.0",
"css-what": "^6.1.0",
"domhandler": "^5.0.2",
"domutils": "^3.0.1",
"nth-check": "^2.0.1"
},
"funding": {
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/css-tree": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.1.3.tgz",
"integrity": "sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==",
"license": "MIT",
"dependencies": {
"mdn-data": "2.0.14",
"source-map": "^0.6.1"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/css-tree/node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/css-what": {
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
"integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
"license": "BSD-2-Clause",
"engines": {
"node": ">= 6"
},
"funding": {
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/csstype": {
"version": "3.2.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
@@ -5609,6 +5666,61 @@
"node": ">=8"
}
},
"node_modules/dom-serializer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"license": "MIT",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"license": "BSD-2-Clause"
},
"node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"license": "BSD-2-Clause",
"dependencies": {
"domelementtype": "^2.3.0"
},
"engines": {
"node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/domutils": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
"license": "BSD-2-Clause",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dotenv": {
"version": "16.4.7",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
@@ -5692,6 +5804,18 @@
"once": "^1.4.0"
}
},
"node_modules/entities": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/env-editor": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/env-editor/-/env-editor-0.4.2.tgz",
@@ -8044,6 +8168,12 @@
"node": ">=0.10"
}
},
"node_modules/mdn-data": {
"version": "2.0.14",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.14.tgz",
"integrity": "sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==",
"license": "CC0-1.0"
},
"node_modules/memoize-one": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
@@ -8834,6 +8964,18 @@
"node": ">=4"
}
},
"node_modules/nth-check": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
"license": "BSD-2-Clause",
"dependencies": {
"boolbase": "^1.0.0"
},
"funding": {
"url": "https://github.com/fb55/nth-check?sponsor=1"
}
},
"node_modules/nullthrows": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz",
@@ -9832,6 +9974,21 @@
"react-native": "*"
}
},
"node_modules/react-native-svg": {
"version": "15.15.2",
"resolved": "https://registry.npmjs.org/react-native-svg/-/react-native-svg-15.15.2.tgz",
"integrity": "sha512-lpaSwA2i+eLvcEdDZyGgMEInQW99K06zjJqfMFblE0yxI0SCN5E4x6in46f0IYi6i3w2t2aaq3oOnyYBe+bo4w==",
"license": "MIT",
"dependencies": {
"css-select": "^5.1.0",
"css-tree": "^1.1.3",
"warn-once": "0.1.1"
},
"peerDependencies": {
"react": "*",
"react-native": "*"
}
},
"node_modules/react-native-view-shot": {
"version": "3.8.0",
"resolved": "https://registry.npmjs.org/react-native-view-shot/-/react-native-view-shot-3.8.0.tgz",

View File

@@ -37,6 +37,7 @@
"react-native-reanimated": "~3.16.1",
"react-native-safe-area-context": "4.12.0",
"react-native-screens": "~4.4.0",
"react-native-svg": "^15.15.2",
"react-native-view-shot": "^3.8.0",
"react-native-web": "~0.19.13",
"readable-stream": "^4.7.0",

View File

@@ -0,0 +1,85 @@
/**
* FlowPuppetSlot - Slot for FlowScreen to show interactive AI puppet.
* Composes PuppetView and optional action buttons; does not depend on FlowScreen logic.
*/
import React, { useState, useCallback } from 'react';
import { View, Text, StyleSheet, TouchableOpacity } from 'react-native';
import { PuppetView } from './PuppetView';
import type { FlowPuppetSlotProps, PuppetAction } from './types';
import { colors } from '../../theme/colors';
import { borderRadius, spacing } from '../../theme/colors';
const ACTIONS: PuppetAction[] = ['smile', 'jump', 'shake'];
export function FlowPuppetSlot({
currentAction,
isTalking,
onAction,
showActionButtons = true,
}: FlowPuppetSlotProps) {
const [localAction, setLocalAction] = useState<PuppetAction>(currentAction);
const effectiveAction = currentAction !== 'idle' ? currentAction : localAction;
const handleAction = useCallback(
(action: PuppetAction) => {
setLocalAction(action);
onAction?.(action);
if (['smile', 'wave', 'nod', 'shake', 'jump'].includes(action)) {
setTimeout(() => {
setLocalAction((prev) => (prev === action ? 'idle' : prev));
onAction?.('idle');
}, 2600);
}
},
[onAction]
);
return (
<View style={styles.wrapper}>
<PuppetView action={effectiveAction} isTalking={isTalking} />
{showActionButtons && (
<View style={styles.actions}>
{ACTIONS.map((act) => (
<TouchableOpacity
key={act}
style={styles.actionBtn}
onPress={() => handleAction(act)}
activeOpacity={0.8}
>
<Text style={styles.actionLabel}>{act}</Text>
</TouchableOpacity>
))}
</View>
)}
</View>
);
}
const styles = StyleSheet.create({
wrapper: {
alignItems: 'center',
justifyContent: 'center',
paddingVertical: spacing.lg,
},
actions: {
flexDirection: 'row',
marginTop: spacing.lg,
gap: spacing.sm,
},
actionBtn: {
paddingHorizontal: spacing.md,
paddingVertical: spacing.sm,
borderRadius: borderRadius.lg,
backgroundColor: colors.flow.cardBackground,
borderWidth: 1,
borderColor: colors.flow.cardBorder,
},
actionLabel: {
fontSize: 12,
fontWeight: '600',
color: colors.flow.primary,
textTransform: 'capitalize',
},
});

View File

@@ -0,0 +1,340 @@
/**
* PuppetView - Interactive blue spirit avatar (React Native).
* Port of airi---interactive-ai-puppet Puppet with same actions:
* idle, wave, nod, shake, jump, think; mouth reflects isTalking.
* Code isolated so FlowScreen stays unchanged except composition.
*/
import React, { useEffect, useRef } from 'react';
import { View, StyleSheet, Animated, Easing } from 'react-native';
import { PuppetViewProps } from './types';
const PUPPET_SIZE = 160;
export function PuppetView({ action, isTalking }: PuppetViewProps) {
const floatAnim = useRef(new Animated.Value(0)).current;
const bounceAnim = useRef(new Animated.Value(0)).current;
const shakeAnim = useRef(new Animated.Value(0)).current;
const thinkScale = useRef(new Animated.Value(1)).current;
const thinkOpacity = useRef(new Animated.Value(1)).current;
const smileScale = useRef(new Animated.Value(1)).current;
// Idle: gentle float
useEffect(() => {
if (action !== 'idle') return;
const loop = Animated.loop(
Animated.sequence([
Animated.timing(floatAnim, {
toValue: 1,
duration: 2000,
useNativeDriver: true,
easing: Easing.inOut(Easing.ease),
}),
Animated.timing(floatAnim, {
toValue: 0,
duration: 2000,
useNativeDriver: true,
easing: Easing.inOut(Easing.ease),
}),
])
);
loop.start();
return () => loop.stop();
}, [action, floatAnim]);
// Smile: exaggerated smile scale pulse
useEffect(() => {
if (action !== 'smile') {
smileScale.setValue(1);
return;
}
const loop = Animated.loop(
Animated.sequence([
Animated.timing(smileScale, {
toValue: 1.12,
duration: 400,
useNativeDriver: true,
easing: Easing.out(Easing.ease),
}),
Animated.timing(smileScale, {
toValue: 1,
duration: 400,
useNativeDriver: true,
easing: Easing.in(Easing.ease),
}),
]),
{ iterations: 3 }
);
loop.start();
return () => loop.stop();
}, [action, smileScale]);
// Wave / Jump: bounce
useEffect(() => {
if (action !== 'wave' && action !== 'jump') return;
const loop = Animated.loop(
Animated.sequence([
Animated.timing(bounceAnim, {
toValue: 1,
duration: 400,
useNativeDriver: true,
easing: Easing.out(Easing.ease),
}),
Animated.timing(bounceAnim, {
toValue: 0,
duration: 400,
useNativeDriver: true,
easing: Easing.in(Easing.ease),
}),
])
);
loop.start();
return () => loop.stop();
}, [action, bounceAnim]);
// Shake: wiggle
useEffect(() => {
if (action !== 'shake') return;
const loop = Animated.loop(
Animated.sequence([
Animated.timing(shakeAnim, {
toValue: 1,
duration: 150,
useNativeDriver: true,
}),
Animated.timing(shakeAnim, {
toValue: 0,
duration: 150,
useNativeDriver: true,
}),
])
);
loop.start();
return () => loop.stop();
}, [action, shakeAnim]);
// Think: scale + opacity pulse
useEffect(() => {
if (action !== 'think') {
thinkScale.setValue(1);
thinkOpacity.setValue(1);
return;
}
const loop = Animated.loop(
Animated.sequence([
Animated.parallel([
Animated.timing(thinkScale, {
toValue: 0.92,
duration: 600,
useNativeDriver: true,
easing: Easing.inOut(Easing.ease),
}),
Animated.timing(thinkOpacity, {
toValue: 0.85,
duration: 600,
useNativeDriver: true,
}),
]),
Animated.parallel([
Animated.timing(thinkScale, {
toValue: 1,
duration: 600,
useNativeDriver: true,
easing: Easing.inOut(Easing.ease),
}),
Animated.timing(thinkOpacity, {
toValue: 1,
duration: 600,
useNativeDriver: true,
}),
]),
])
);
loop.start();
return () => loop.stop();
}, [action, thinkScale, thinkOpacity]);
const floatY = floatAnim.interpolate({
inputRange: [0, 1],
outputRange: [0, -8],
});
const bounceY = bounceAnim.interpolate({
inputRange: [0, 1],
outputRange: [0, -20],
});
const shakeRotate = shakeAnim.interpolate({
inputRange: [0, 1],
outputRange: ['0deg', '8deg'],
});
const isBounce = action === 'wave' || action === 'jump';
const isShake = action === 'shake';
const isSmile = action === 'smile';
const mouthStyle = isTalking
? [styles.mouth, styles.mouthOpen]
: isSmile
? [styles.mouth, styles.mouthBigSmile]
: [styles.mouth, styles.mouthSmile];
return (
<Animated.View
style={[
styles.container,
action === 'idle' && {
transform: [{ translateY: floatY }],
},
isBounce && {
transform: [{ translateY: bounceY }],
},
isShake && {
transform: [{ rotate: shakeRotate }],
},
action === 'think' && {
transform: [{ scale: thinkScale }],
opacity: thinkOpacity,
},
isSmile && {
transform: [{ scale: smileScale }],
},
]}
>
{/* Aura glow */}
<View style={styles.aura} />
{/* Body (droplet-like rounded rect) */}
<View style={styles.body}>
{/* Gloss */}
<View style={styles.gloss} />
{/* Cheeks */}
<View style={[styles.cheek, styles.cheekLeft]} />
<View style={[styles.cheek, styles.cheekRight]} />
{/* Eyes */}
<View style={styles.eyes}>
<View style={[styles.eye, styles.eyeLeft]}>
<View style={styles.eyeSparkle} />
</View>
<View style={[styles.eye, styles.eyeRight]}>
<View style={styles.eyeSparkle} />
</View>
</View>
{/* Mouth - default smile; open when talking; big smile when smile action */}
<View style={mouthStyle} />
</View>
</Animated.View>
);
}
const BODY_SIZE = PUPPET_SIZE * 0.9;
const EYE_SIZE = 10;
const EYE_OFFSET_X = 18;
const EYE_OFFSET_Y = -8;
const styles = StyleSheet.create({
container: {
width: PUPPET_SIZE,
height: PUPPET_SIZE,
alignItems: 'center',
justifyContent: 'center',
},
aura: {
position: 'absolute',
width: PUPPET_SIZE + 40,
height: PUPPET_SIZE + 40,
borderRadius: (PUPPET_SIZE + 40) / 2,
backgroundColor: 'rgba(14, 165, 233, 0.15)',
},
body: {
width: BODY_SIZE,
height: BODY_SIZE,
borderRadius: BODY_SIZE / 2,
backgroundColor: '#0ea5e9',
alignItems: 'center',
justifyContent: 'center',
borderWidth: 0,
overflow: 'hidden',
shadowColor: '#0c4a6e',
shadowOffset: { width: 0, height: 8 },
shadowOpacity: 0.35,
shadowRadius: 12,
elevation: 8,
},
gloss: {
position: 'absolute',
top: BODY_SIZE * 0.12,
left: BODY_SIZE * 0.2,
right: BODY_SIZE * 0.2,
height: 4,
borderRadius: 2,
backgroundColor: 'rgba(255,255,255,0.35)',
},
cheek: {
position: 'absolute',
width: 14,
height: 14,
borderRadius: 7,
backgroundColor: 'rgba(59, 130, 246, 0.35)',
},
cheekLeft: {
left: BODY_SIZE * 0.15,
top: BODY_SIZE * 0.42,
},
cheekRight: {
right: BODY_SIZE * 0.15,
top: BODY_SIZE * 0.42,
},
eyes: {
flexDirection: 'row',
alignItems: 'center',
justifyContent: 'center',
position: 'absolute',
top: BODY_SIZE * 0.34,
},
eye: {
width: EYE_SIZE,
height: EYE_SIZE,
borderRadius: EYE_SIZE / 2,
backgroundColor: '#0c4a6e',
alignItems: 'center',
justifyContent: 'center',
},
eyeLeft: { marginRight: EYE_OFFSET_X },
eyeRight: { marginLeft: EYE_OFFSET_X },
eyeSparkle: {
width: 3,
height: 3,
borderRadius: 1.5,
backgroundColor: '#fff',
position: 'absolute',
top: 1,
left: 2,
},
mouth: {
position: 'absolute',
top: BODY_SIZE * 0.52,
backgroundColor: '#0c4a6e',
},
mouthSmile: {
width: 22,
height: 6,
borderBottomLeftRadius: 11,
borderBottomRightRadius: 11,
borderTopLeftRadius: 0,
borderTopRightRadius: 0,
},
mouthOpen: {
width: 18,
height: 6,
top: BODY_SIZE * 0.51,
borderRadius: 3,
backgroundColor: 'rgba(12, 74, 110, 0.9)',
},
mouthBigSmile: {
width: 32,
height: 10,
top: BODY_SIZE * 0.51,
borderBottomLeftRadius: 16,
borderBottomRightRadius: 16,
borderTopLeftRadius: 0,
borderTopRightRadius: 0,
},
});

View File

@@ -0,0 +1,3 @@
export { PuppetView } from './PuppetView';
export { FlowPuppetSlot } from './FlowPuppetSlot';
export type { PuppetAction, PuppetState, PuppetViewProps, FlowPuppetSlotProps } from './types';

View File

@@ -0,0 +1,28 @@
/**
* Puppet types - compatible with airi interactive AI puppet semantics.
* Used for FlowScreen multimodal avatar (action + talking state).
*/
export type PuppetAction = 'idle' | 'wave' | 'nod' | 'shake' | 'jump' | 'think' | 'talk' | 'smile';
export interface PuppetState {
currentAction: PuppetAction;
isTalking: boolean;
isThinking: boolean;
}
export interface PuppetViewProps {
action: PuppetAction;
isTalking: boolean;
}
export interface FlowPuppetSlotProps {
/** Current action (idle, wave, nod, shake, jump, think). */
currentAction: PuppetAction;
/** True when AI is "speaking" (e.g. streaming or responding). */
isTalking: boolean;
/** Optional: allow parent to set action (e.g. from AI tool call). */
onAction?: (action: PuppetAction) => void;
/** Show quick action buttons (wave, jump, shake) for interactivity. */
showActionButtons?: boolean;
}

View File

@@ -40,6 +40,8 @@ import { storageService } from '../services/storage.service';
import AsyncStorage from '@react-native-async-storage/async-storage';
import { SentinelVault } from '../utils/crypto_core';
import { Buffer } from 'buffer';
import { FlowPuppetSlot } from '../components/puppet';
import type { PuppetAction } from '../components/puppet';
// =============================================================================
// Type Definitions
@@ -97,6 +99,9 @@ export default function FlowScreen() {
const [saveResult, setSaveResult] = useState<{ success: boolean; message: string }>({ success: true, message: '' });
const [isSavingToVault, setIsSavingToVault] = useState(false);
// AI multimodal puppet (optional; does not affect existing chat logic)
const [puppetAction, setPuppetAction] = useState<PuppetAction>('idle');
const [chatHistory, setChatHistory] = useState<ChatSession[]>([
// Sample history data
{
@@ -207,6 +212,12 @@ export default function FlowScreen() {
}
}, [aiRoles]);
// Sync puppet action with sending state (think while AI is responding)
useEffect(() => {
if (isSending) setPuppetAction('think');
else setPuppetAction((prev) => (prev === 'think' ? 'idle' : prev));
}, [isSending]);
// Save current messages for the active role when they change
useEffect(() => {
if (user && selectedRole && messages.length >= 0) { // Save even if empty to allow clearing
@@ -784,6 +795,14 @@ export default function FlowScreen() {
</TouchableOpacity>
</View>
{/* AI multimodal puppet (optional slot; code in components/puppet) */}
<FlowPuppetSlot
currentAction={puppetAction}
isTalking={isSending}
onAction={setPuppetAction}
showActionButtons={true}
/>
{/* Chat Messages */}
<ScrollView
ref={scrollViewRef}