基于 ESP32-S3 + VB6824 的四博 AI 双目交互终端设计：从双目动画到多模态事件系统

基于 ESP32-S3 + VB6824 的四博 AI 双目交互终端设计：从双目动画到多模态事件系统

1. 项目背景

AI 硬件如果只停留在“语音问答”，用户体验会比较单薄。真正有产品感的 AI 终端，需要具备完整的多模态交互能力：

语音输入：唤醒、录音、打断 视觉输出：双目动画、状态表情、情绪反馈 触摸输入：顶部、左右、背部多点触控 姿态输入：摇晃、翻转、倾斜、拿起 触觉输出：震动确认、错误提醒、OTA节奏反馈 云端能力：ASR / LLM / TTS / 知识库 / MCP 小程序能力：配网、声音克隆、知识库、素材与固件更新

四博 AI-S3 双目方案采用 ESP32-S3 主控，配合 VB6824 语音前端和双目屏，可以对接小智、豆包、ChatGPT 等主流大模型；资料中也提到 AI-S3 双目双屏方案主要配件包括 1.28 寸 LCD 显示屏 x2、主板、麦克风、喇叭和线材，选用 ESP32S3R8 + 16M Flash + VB6824，并支持二次开发。

本文从固件工程角度，设计一套适合 AI 双目音箱、AI 桌宠、AI 陪伴机器人、儿童学习终端和 IP 潮玩设备的嵌入式软件架构。

2. 硬件架构

四博 AI 双目可以抽象为 6 个硬件子系统：

1. 主控系统：ESP32-S3R8 / ESP32-S3R2 2. 语音系统：VB6824 + Mic + Speaker + AMP 3. 显示系统：0.71 / 1.28 寸双目屏 4. 交互系统：四路触摸 / 按键 / 震动马达 5. 姿态系统：三轴加速度计 / 六轴 IMU 6. 平台系统：四博小助手 + 云端 AI + MCP + OTA

资料中双目主板支持 0.71 寸和 1.28 寸屏，V1/V2 版本对屏幕类型和调光能力有不同支持；QA 中也给出 1.28 寸屏为 240×240，0.71 寸屏为 160×160。

推荐配置如下：

3. 软件总体架构

AI 双目设备不建议采用单一while(1)逻辑，而应采用FreeRTOS 多任务 + 事件队列 + 状态机的方式。

推荐任务划分：

app_main ├── wifi_task // Wi-Fi连接、断线重连 ├── blufi_task // 四博小助手 BLE 配网 ├── vb6824_task // 唤醒、打断、录音事件 ├── ai_ws_task // WebSocket / MQTT AI通信 ├── eye_render_task // 双目动画刷新 ├── touch_scan_task // 四路触摸扫描 ├── imu_detect_task // 姿态检测 ├── haptic_task // 震动马达控制 ├── mcp_dispatch_task // AI工具调用 ├── ota_task // 固件和素材升级 └── app_dispatch_task // 全局事件分发

工程目录建议：

sibo_ai_eye/ ├── main/ │ ├── app_main.c │ ├── app_event.c │ ├── app_state.c │ ├── app_dispatch.c │ └── board_config.h ├── components/ │ ├── eye_display/ │ ├── touch_input/ │ ├── imu_sensor/ │ ├── haptic_motor/ │ ├── vb6824/ │ ├── ai_client/ │ ├── mcp_tools/ │ ├── ota_manager/ │ ├── asset_manager/ │ └── factory_test/ ├── partitions.csv └── sdkconfig.defaults

4. 板级配置抽象

因为 AI 双目可能有 0.71 寸、1.28 寸、1.8 寸等不同版本，建议用board_config.h统一管理。

#pragma once #define BOARD_NAME "SIBO_AI_EYE_PRO" #define BOARD_VERSION "V1.0" #define CONFIG_USE_ESP32S3 1 #define CONFIG_USE_VB6824 1 #define CONFIG_EYE_SCREEN_128 1 #define CONFIG_EYE_SCREEN_071 0 #define CONFIG_USE_TOUCH_4CH 1 #define CONFIG_USE_IMU 1 #define CONFIG_USE_HAPTIC 1 #define CONFIG_USE_BATTERY 1 #define MOTOR_GPIO 15 #define IMU_I2C_PORT I2C_NUM_0 #define IMU_SDA_GPIO 8 #define IMU_SCL_GPIO 9 #define VB6824_UART UART_NUM_1 #define VB6824_TX_GPIO 17 #define VB6824_RX_GPIO 18 #define VB6824_BAUD 115200

屏幕参数抽象：

typedef enum { EYE_SCREEN_TYPE_071 = 0, EYE_SCREEN_TYPE_128, EYE_SCREEN_TYPE_180, } eye_screen_type_t; typedef struct { eye_screen_type_t type; int width; int height; int fps; const char *asset_path; } eye_screen_config_t; eye_screen_config_t board_get_eye_config(void) { #if CONFIG_EYE_SCREEN_128 return (eye_screen_config_t) { .type = EYE_SCREEN_TYPE_128, .width = 240, .height = 240, .fps = 30, .asset_path = "/spiffs/eye_128", }; #else return (eye_screen_config_t) { .type = EYE_SCREEN_TYPE_071, .width = 160, .height = 160, .fps = 24, .asset_path = "/spiffs/eye_071", }; #endif }

这样同一套固件工程可以适配不同屏幕 SKU。

5. 全局事件系统设计

所有输入都转化为事件，由统一状态机分发：

typedef enum { APP_EVT_NONE = 0, APP_EVT_WIFI_CONNECTED, APP_EVT_WIFI_DISCONNECTED, APP_EVT_BLUFI_START, APP_EVT_BLUFI_DONE, APP_EVT_WAKE_WORD, APP_EVT_VOICE_INTERRUPT, APP_EVT_RECORD_START, APP_EVT_RECORD_STOP, APP_EVT_TOUCH_TOP, APP_EVT_TOUCH_LEFT, APP_EVT_TOUCH_RIGHT, APP_EVT_TOUCH_BACK, APP_EVT_IMU_PICKUP, APP_EVT_IMU_SHAKE, APP_EVT_IMU_FLIP, APP_EVT_IMU_TILT_LEFT, APP_EVT_IMU_TILT_RIGHT, APP_EVT_AI_LISTENING, APP_EVT_AI_THINKING, APP_EVT_AI_SPEAKING, APP_EVT_AI_FINISHED, APP_EVT_AI_ERROR, APP_EVT_MCP_TOOL_CALL, APP_EVT_LOW_BATTERY, APP_EVT_OTA_START, APP_EVT_OTA_DONE, } app_event_id_t; typedef struct { app_event_id_t id; int param1; int param2; char payload[256]; } app_event_t; static QueueHandle_t g_app_event_queue;

事件投递函数：

void app_post_event(app_event_id_t id, int p1, int p2, const char *payload) { if (!g_app_event_queue) { return; } app_event_t evt = { .id = id, .param1 = p1, .param2 = p2, }; if (payload) { strncpy(evt.payload, payload, sizeof(evt.payload) - 1); } xQueueSend(g_app_event_queue, &evt, 0); }

主程序入口：

void app_main(void) { ESP_ERROR_CHECK(nvs_flash_init()); g_app_event_queue = xQueueCreate(32, sizeof(app_event_t)); if (!g_app_event_queue) { ESP_LOGE("APP", "事件队列创建失败"); return; } eye_app_start(); haptic_app_start(); touch_app_start(); imu_app_start(); wifi_app_start(); blufi_app_start(); vb6824_app_start(); ai_client_start(); mcp_app_start(); ota_app_start(); app_dispatch_start(); ESP_LOGI("APP", "四博 AI 双目系统启动完成"); }

6. AI 状态机设计

状态机负责将双目动画、音频、震动和联网状态统一起来。

typedef enum { AI_STATE_BOOT = 0, AI_STATE_IDLE, AI_STATE_WAKEUP, AI_STATE_LISTENING, AI_STATE_THINKING, AI_STATE_SPEAKING, AI_STATE_TOUCH_FEEDBACK, AI_STATE_SHAKE_FEEDBACK, AI_STATE_SLEEP, AI_STATE_NETWORK_ERROR, AI_STATE_LOW_BATTERY, AI_STATE_OTA, } ai_state_t; static ai_state_t g_ai_state = AI_STATE_BOOT;

状态切换：

void ai_set_state(ai_state_t state) { g_ai_state = state; switch (state) { case AI_STATE_BOOT: eye_play_anim("boot"); audio_play_prompt("boot.wav"); break; case AI_STATE_IDLE: eye_play_anim("idle_blink"); break; case AI_STATE_WAKEUP: eye_play_anim("wake"); haptic_double(); audio_play_prompt("ding.wav"); break; case AI_STATE_LISTENING: eye_play_anim("listening"); audio_start_record(); break; case AI_STATE_THINKING: eye_play_anim("thinking"); break; case AI_STATE_SPEAKING: eye_play_anim("speaking"); break; case AI_STATE_TOUCH_FEEDBACK: eye_play_anim("happy"); haptic_short(); break; case AI_STATE_SHAKE_FEEDBACK: eye_play_anim("surprised"); haptic_double(); break; case AI_STATE_SLEEP: eye_play_anim("sleep"); audio_stop_tts(); break; case AI_STATE_NETWORK_ERROR: eye_play_anim("net_error"); haptic_long(); audio_play_prompt("network_error.wav"); break; case AI_STATE_LOW_BATTERY: eye_play_anim("low_battery"); haptic_long(); audio_play_prompt("low_battery.wav"); break; case AI_STATE_OTA: eye_play_anim("ota"); haptic_rhythm_ota(); break; default: eye_play_anim("idle_blink"); break; } }

7. 四路触摸输入

四路触摸建议映射为：

触摸检测：

#include "driver/touch_sensor.h" #define TOUCH_THRESHOLD_PERCENT 70 typedef struct { touch_pad_t pad; uint32_t baseline; app_event_id_t evt; const char *name; } touch_key_t; static touch_key_t s_touch_keys[] = { {TOUCH_PAD_NUM1, 0, APP_EVT_TOUCH_TOP, "TOP"}, {TOUCH_PAD_NUM2, 0, APP_EVT_TOUCH_LEFT, "LEFT"}, {TOUCH_PAD_NUM3, 0, APP_EVT_TOUCH_RIGHT, "RIGHT"}, {TOUCH_PAD_NUM4, 0, APP_EVT_TOUCH_BACK, "BACK"}, };

触摸基线校准：

static void touch_calibrate(void) { for (int i = 0; i < 4; i++) { uint32_t sum = 0; for (int j = 0; j < 20; j++) { uint32_t raw = 0; touch_pad_read_raw_data(s_touch_keys[i].pad, &raw); sum += raw; vTaskDelay(pdMS_TO_TICKS(10)); } s_touch_keys[i].baseline = sum / 20; ESP_LOGI("TOUCH", "%s baseline=%lu", s_touch_keys[i].name, s_touch_keys[i].baseline); } }

触摸扫描任务：

static void touch_task(void *arg) { while (1) { for (int i = 0; i < 4; i++) { uint32_t raw = 0; touch_pad_read_raw_data(s_touch_keys[i].pad, &raw); uint32_t threshold = s_touch_keys[i].baseline * TOUCH_THRESHOLD_PERCENT / 100; if (raw < threshold) { ESP_LOGI("TOUCH", "%s touched raw=%lu", s_touch_keys[i].name, raw); app_post_event(s_touch_keys[i].evt, raw, 0, NULL); vTaskDelay(pdMS_TO_TICKS(250)); } } vTaskDelay(pdMS_TO_TICKS(30)); } } void touch_app_start(void) { ESP_ERROR_CHECK(touch_pad_init()); for (int i = 0; i < 4; i++) { ESP_ERROR_CHECK(touch_pad_config(s_touch_keys[i].pad)); } vTaskDelay(pdMS_TO_TICKS(300)); touch_calibrate(); xTaskCreate(touch_task, "touch_task", 4096, NULL, 5, NULL); }

8. 三轴姿态识别

三轴传感器可实现摇晃、翻转、倾斜和拿起唤醒。

typedef struct { int16_t x; int16_t y; int16_t z; } accel_data_t;

I2C 读取：

#define I2C_PORT I2C_NUM_0 #define I2C_SDA_GPIO 8 #define I2C_SCL_GPIO 9 #define I2C_FREQ_HZ 400000 #define IMU_ADDR 0x68 #define IMU_REG_ACCEL_X 0x3B static esp_err_t imu_read_accel(accel_data_t *acc) { uint8_t reg = IMU_REG_ACCEL_X; uint8_t buf[6] = {0}; esp_err_t ret = i2c_master_write_read_device( I2C_PORT, IMU_ADDR, &reg, 1, buf, sizeof(buf), pdMS_TO_TICKS(50) ); if (ret != ESP_OK) { return ret; } acc->x = (int16_t)((buf[0] << 8) | buf[1]); acc->y = (int16_t)((buf[2] << 8) | buf[3]); acc->z = (int16_t)((buf[4] << 8) | buf[5]); return ESP_OK; }

动作判断：

static bool imu_detect_shake(accel_data_t now, accel_data_t last) { int dx = abs(now.x - last.x); int dy = abs(now.y - last.y); int dz = abs(now.z - last.z); return (dx + dy + dz) > 18000; } static bool imu_detect_flip(accel_data_t now) { return now.z < -12000; } static bool imu_detect_tilt_left(accel_data_t now) { return now.x < -10000; } static bool imu_detect_tilt_right(accel_data_t now) { return now.x > 10000; }

IMU 任务：

static void imu_task(void *arg) { accel_data_t last = {0}; while (1) { accel_data_t now; if (imu_read_accel(&now) == ESP_OK) { if (imu_detect_shake(now, last)) { app_post_event(APP_EVT_IMU_SHAKE, now.x, now.y, NULL); } if (imu_detect_flip(now)) { app_post_event(APP_EVT_IMU_FLIP, now.x, now.z, NULL); } if (imu_detect_tilt_left(now)) { app_post_event(APP_EVT_IMU_TILT_LEFT, now.x, 0, NULL); } if (imu_detect_tilt_right(now)) { app_post_event(APP_EVT_IMU_TILT_RIGHT, now.x, 0, NULL); } last = now; } vTaskDelay(pdMS_TO_TICKS(80)); } }

9. VB6824 语音事件解析

资料中 ESP32-C2/C3/S3 + VB6824 方案由 VB6824 完成音频编解码、AEC、语音唤醒和改唤醒词等功能，让主控专注于通信和 UI；该方案已经用于电子吧唧、S3 双目、S3 拍学机、地球仪、拍拍灯等产品。

建议 VB6824 与 ESP32-S3 之间通过 UART 上报事件：

0xA5 0x01 0x00 0x5A 唤醒成功 0xA5 0x02 0x00 0x5A 用户打断 0xA5 0x03 0x00 0x5A 开始录音 0xA5 0x04 0x00 0x5A 停止录音

代码：

#define VB_UART_NUM UART_NUM_1 #define VB_UART_TX 17 #define VB_UART_RX 18 #define VB_UART_BAUD 115200 #define VB_HEAD 0xA5 #define VB_TAIL 0x5A #define VB_CMD_WAKEUP 0x01 #define VB_CMD_INTERRUPT 0x02 #define VB_CMD_REC_START 0x03 #define VB_CMD_REC_STOP 0x04 static void vb6824_parse_frame(uint8_t *buf, int len) { if (len < 4) { return; } if (buf[0] != VB_HEAD || buf[3] != VB_TAIL) { return; } switch (buf[1]) { case VB_CMD_WAKEUP: app_post_event(APP_EVT_WAKE_WORD, 0, 0, NULL); break; case VB_CMD_INTERRUPT: app_post_event(APP_EVT_VOICE_INTERRUPT, 0, 0, NULL); break; case VB_CMD_REC_START: app_post_event(APP_EVT_RECORD_START, 0, 0, NULL); break; case VB_CMD_REC_STOP: app_post_event(APP_EVT_RECORD_STOP, 0, 0, NULL); break; default: break; } }

10. 统一事件分发

static void app_dispatch_task(void *arg) { app_event_t evt; while (1) { if (xQueueReceive(g_app_event_queue, &evt, portMAX_DELAY)) { switch (evt.id) { case APP_EVT_WIFI_CONNECTED: ai_set_state(AI_STATE_IDLE); break; case APP_EVT_WIFI_DISCONNECTED: ai_set_state(AI_STATE_NETWORK_ERROR); break; case APP_EVT_WAKE_WORD: ai_set_state(AI_STATE_WAKEUP); ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_VOICE_INTERRUPT: audio_stop_tts(); ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_RECORD_START: ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_RECORD_STOP: audio_stop_record(); ai_set_state(AI_STATE_THINKING); break; case APP_EVT_TOUCH_TOP: ai_set_state(AI_STATE_TOUCH_FEEDBACK); app_post_event(APP_EVT_WAKE_WORD, 0, 0, NULL); break; case APP_EVT_TOUCH_LEFT: role_switch_prev(); eye_play_anim("prev_role"); haptic_short(); break; case APP_EVT_TOUCH_RIGHT: role_switch_next(); eye_play_anim("next_role"); haptic_short(); break; case APP_EVT_TOUCH_BACK: eye_play_anim("config"); haptic_double(); blufi_start_config(); break; case APP_EVT_IMU_SHAKE: ai_set_state(AI_STATE_SHAKE_FEEDBACK); role_switch_random(); break; case APP_EVT_IMU_FLIP: ai_set_state(AI_STATE_SLEEP); audio_stop_tts(); break; case APP_EVT_IMU_TILT_LEFT: eye_set_gaze(-30, 0); break; case APP_EVT_IMU_TILT_RIGHT: eye_set_gaze(30, 0); break; case APP_EVT_AI_THINKING: ai_set_state(AI_STATE_THINKING); break; case APP_EVT_AI_SPEAKING: ai_set_state(AI_STATE_SPEAKING); break; case APP_EVT_AI_ERROR: ai_set_state(AI_STATE_NETWORK_ERROR); break; case APP_EVT_OTA_START: ai_set_state(AI_STATE_OTA); break; default: break; } } } }

11. 眼睛素材与 OTA 设计

资料中提到，0.71/1.28 寸双目固件可通过修改screen_type指向屏幕类型，并替换defaultEye_1.28.h或defaultEye_0.71.h素材数组来定制眼睛样式。

建议不要把所有眼睛动画写死到固件里，而是放到素材分区：

/spiffs/ ├── eye_128/ │ ├── idle.bin │ ├── wake.bin │ ├── thinking.bin │ ├── speaking.bin │ └── happy.bin ├── eye_071/ │ ├── idle.bin │ └── happy.bin └── prompt/ ├── boot.wav ├── ding.wav └── net_error.wav

动画加载：

typedef struct { char name[32]; int frame_count; int width; int height; int fps; FILE *fp; } eye_anim_t; static eye_anim_t s_current_anim; esp_err_t eye_anim_open(const char *name) { char path[96]; eye_screen_config_t cfg = board_get_eye_config(); snprintf(path, sizeof(path), "%s/%s.bin", cfg.asset_path, name); FILE *fp = fopen(path, "rb"); if (!fp) { ESP_LOGW("EYE", "动画文件不存在: %s", path); return ESP_FAIL; } memset(&s_current_anim, 0, sizeof(s_current_anim)); strncpy(s_current_anim.name, name, sizeof(s_current_anim.name) - 1); s_current_anim.fp = fp; fread(&s_current_anim.frame_count, sizeof(int), 1, fp); fread(&s_current_anim.width, sizeof(int), 1, fp); fread(&s_current_anim.height, sizeof(int), 1, fp); fread(&s_current_anim.fps, sizeof(int), 1, fp); return ESP_OK; } void eye_play_anim(const char *name) { if (s_current_anim.fp) { fclose(s_current_anim.fp); s_current_anim.fp = NULL; } if (eye_anim_open(name) != ESP_OK) { eye_anim_open("idle"); } }

推荐分区表：

# Name, Type, SubType, Offset, Size nvs, data, nvs, 0x9000, 0x6000 otadata, data, ota, 0xf000, 0x2000 phy_init, data, phy, 0x11000, 0x1000 factory, app, factory, 0x20000, 2M ota_0, app, ota_0, 0x220000, 2M ota_1, app, ota_1, 0x420000, 2M assets, data, spiffs, 0x620000, 4M storage, data, fat, 0xA20000, 4M

12. MCP 工具扩展

资料中 MCP 管理指令支持AT+ADDMCP，可配置不返回 AI 参数或返回 AI 参数的工具调用；示例中包括设置屏幕主题、设置 RGB 灯光颜色，并规定了对应串口返回协议。

定义一个双目表情工具：

{ "name": "self.eye.set_expression", "description": "设置四博AI双目的表情状态", "parameters": { "type": "object", "properties": { "expression": { "type": "string", "enum": ["开心", "思考", "惊讶", "困惑", "睡觉", "待机"] } }, "required": ["expression"] } }

设备侧处理：

void mcp_set_eye_expression(const char *expression) { if (strcmp(expression, "开心") == 0) { eye_play_anim("happy"); haptic_short(); } else if (strcmp(expression, "思考") == 0) { eye_play_anim("thinking"); } else if (strcmp(expression, "惊讶") == 0) { eye_play_anim("surprised"); haptic_double(); } else if (strcmp(expression, "困惑") == 0) { eye_play_anim("confused"); } else if (strcmp(expression, "睡觉") == 0) { eye_play_anim("sleep"); } else { eye_play_anim("idle_blink"); } }

工具分发：

void mcp_dispatch_tool(const char *tool_name, cJSON *args) { if (strcmp(tool_name, "self.eye.set_expression") == 0) { cJSON *exp = cJSON_GetObjectItem(args, "expression"); if (cJSON_IsString(exp)) { mcp_set_eye_expression(exp->valuestring); } return; } if (strcmp(tool_name, "self.device.enter_config") == 0) { blufi_start_config(); eye_play_anim("config"); haptic_double(); return; } if (strcmp(tool_name, "self.audio.stop") == 0) { audio_stop_tts(); ai_set_state(AI_STATE_IDLE); return; } }

13. 产测设计

AI 双目量产建议准备独立 factory 固件，覆盖：

screen 双目屏 RGB、坏点、左右屏同步 touch 四路触摸原始值、阈值 motor 短震、双震、长震 imu X/Y/Z 方向、摇晃、翻转 mic 录音电平、底噪 speaker 提示音播放 vb6824 唤醒、打断、升级模式 wifi RSSI、连接、重连 flash NVS、OTA、素材分区

产测代码：

void factory_reply(const char *item, const char *result) { printf("{\"item\":\"%s\",\"result\":\"%s\"}\n", item, result); } void factory_test_screen(void) { lcd_fill_color(0xF800); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x07E0); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x001F); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x0000); factory_reply("screen", "ok"); } void factory_test_touch(void) { int top = touch_get_raw(0); int left = touch_get_raw(1); int right = touch_get_raw(2); int back = touch_get_raw(3); printf("{\"item\":\"touch\",\"result\":\"ok\"," "\"top\":%d,\"left\":%d,\"right\":%d,\"back\":%d}\n", top, left, right, back); } void factory_test_imu(void) { accel_data_t acc; if (imu_read_accel(&acc) == ESP_OK) { printf("{\"item\":\"imu\",\"result\":\"ok\"," "\"x\":%d,\"y\":%d,\"z\":%d}\n", acc.x, acc.y, acc.z); } else { factory_reply("imu", "fail"); } }

14. 总结

四博 AI 双目方案可以理解为一个多模态 AI 终端底座：

ESP32-S3：负责联网、UI、状态机、AI通信 VB6824：负责唤醒、AEC、打断、语音前端 双目屏：负责表情显示和状态可视化 触摸节点：负责本地输入 IMU：负责姿态感知 震动马达：负责触觉反馈 四博小助手：负责配网、声音克隆、知识库、MCP、OTA

它不仅适合 AI 音箱，也适合 AI 桌宠、儿童陪伴机、学习终端、IP 潮玩和品牌定制设备。真正的 AI 硬件不只是“会回答问题”，而是要在声音、表情、触摸、姿态、触觉和云端智能之间形成完整闭环。