Real-time digital-human interaction protocol

// WebSocket connection example
const protocol = 'webrtc'; // or 'rtmp'
const avatorId = 'your-avator-id';
const bizId = '0'; // or 'asr' (when ASR is enabled for humanctrl)
const token = 'your-access-token';

const wsUrl = `wss://wsapi.nuwaai.com/websk/humanAgent/humanchat/${protocol}/${avatorId}/${bizId}?access_token=${token}`;
const ws = new WebSocket(wsUrl);

// Connection established
ws.onopen = function() {
  console.log('WebSocket connection established');
};

// Receive messages
ws.onmessage = function(event) {
  const message = JSON.parse(event.data);
  console.log('Message received:', message);
  
  // Handle the connection-verification response
  if (message.type === 'avator-connect') {
    if (message.code === 200) {
      console.log('Connection verified; session ID:', message.data.sessionid);
      console.log('Suggested questions:', message.data.question);
      // After verification succeeds, continue to the next operation
    } else {
      console.error('Connection verification failed:', message.msg);
    }
  }
};

// Connection closed
ws.onclose = function() {
  console.log('WebSocket connection closed');
};

// Connection error
ws.onerror = function(error) {
  console.error('WebSocket connection error:', error);
};

// Send a WebRTC media-negotiation request
function sendPlayStream(ws, localSdp) {
  const playStreamMessage = {
    type: 'playStream',
    data: {
      sdp: localSdp
    }
  };
  
  ws.send(JSON.stringify(playStreamMessage));
  console.log('playStream request sent:', playStreamMessage);
}

// Handle the WebRTC negotiation response
function handleAvatorPlay(message) {
  if (message.code === 200) {
    const remoteSdp = message.data.sdp;
    console.log('avator-play response received; remote SDP:', remoteSdp);
    
    // Establish the WebRTC connection with the remote SDP
    // Handle this according to your WebRTC implementation
    // For example:
    // pc.setRemoteDescription(new RTCSessionDescription({ type: 'answer', sdp: remoteSdp }));
  } else {
    console.error('Media negotiation failed:', message.msg);
  }
}

// Example: add this to the WebSocket message handler
ws.onmessage = function(event) {
  const message = JSON.parse(event.data);
  console.log('Message received:', message);
  
  if (message.type === 'avator-connect') {
    // Handle the connection-verification response...
  } else if (message.type === 'avator-play') {
    // Handle the media-negotiation response
    handleAvatorPlay(message);
  }
};

// Send an RTMP media-negotiation request
function sendPlayRtmp(ws, rtmpUrl) {
  const playRtmpMessage = {
    type: 'playRtmp',
    data: {
      url: rtmpUrl
    }
  };
  
  ws.send(JSON.stringify(playRtmpMessage));
  console.log('playRtmp request sent:', playRtmpMessage);
}

// Handle the RTMP negotiation response
function handleAvatorPlayRtmp(message) {
  if (message.code === 200) {
    console.log('RTMP media negotiation succeeded');
    
    // Begin receiving the audio/video stream over RTMP
    // Handle this according to your RTMP player implementation
    // For example, load the RTMP stream with video.js or another player
  } else {
    console.error('RTMP media negotiation failed:', message.msg);
  }
}

// Example: add this to the WebSocket message handler
ws.onmessage = function(event) {
  const message = JSON.parse(event.data);
  console.log('Message received:', message);
  
  if (message.type === 'avator-connect') {
    // Handle the connection-verification response...
  } else if (message.type === 'avator-play') {
    // Handle the media-negotiation response
    handleAvatorPlayRtmp(message);
  }
};

// Send a text conversation message
function sendTextMessage(ws, text) {
  const textMessage = {
    type: 'agent',
    data: {
      content: text
    }
  };
  
  ws.send(JSON.stringify(textMessage));
  console.log('Text message sent:', textMessage);
}

// Example: send a text message
// sendTextMessage(ws, '你好，数字人！');

// Handle conversation responses
function handleChatMessages(message) {
  switch (message.type) {
    case 'chat_begin':
      console.log('Conversation started');
      break;
    case 'chat':
      console.log('Language-model response:', message.data.content);
      break;
    case 'chat_end':
      console.log('Conversation ended');
      break;
    case 'avator-text':
      console.log('Digital-human response:', message.data.text);
      break;
    case 'avator-playstop':
      console.log('Digital-human response complete');
      break;
  }
}

// Example: add this to the WebSocket message handler
ws.onmessage = function(event) {
  const message = JSON.parse(event.data);
  console.log('Message received:', message);
  
  if (message.type === 'avator-connect') {
    // Handle the connection-verification response...
  } else if (message.type === 'avator-play') {
    // Handle the media-negotiation response...
  } else if (['chat_begin', 'chat', 'chat_end', 'avator-text', 'avator-playstop'].includes(message.type)) {
    // Handle conversation responses
    handleChatMessages(message);
  }
};

// Send a digital-human control message (valid only for humanctrl connections)
function sendAvatarControlMessage(ws, content) {
  const controlMessage = {
    type: 'avatar',
    data: {
      content: content
    }
  };
  
  ws.send(JSON.stringify(controlMessage));
  console.log('Digital-human control message sent:', controlMessage);
}

// Example: send a digital-human control message
// sendAvatarControlMessage(ws, '欢迎使用 Nuwa 数字人服务！');

// Handle digital-human control responses
function handleAvatarControlResponse(message) {
  if (message.type === 'avator-ctrl') {
    console.log('Digital-human control message:', message.data);
  } else if (message.type === 'avator-playstop') {
    console.log('Digital-human response complete');
  }
}

// Example: add this to the WebSocket message handler
ws.onmessage = function(event) {
  const message = JSON.parse(event.data);
  console.log('Message received:', message);
  
  if (message.type === 'avator-connect') {
    // Handle the connection-verification response...
  } else if (message.type === 'avator-ctrl' || message.type === 'avator-playstop') {
    // Handle digital-human control responses
    handleAvatarControlResponse(message);
  }
};

// Send a digital-human interruption message (valid only for humanctrl connections)
function sendInterruptChatMessage(ws) {
  const interrupt_chat_message = {
    type: 'interrupt_chat'
  };
  
  ws.send(JSON.stringify(interrupt_chat_message));
  console.log('Digital-human interruption message sent:', interrupt_chat_message);
}

// WebRTC connection example
let peerConnection = null;
let peerStream = null;

// Start media negotiation
function startMediaNegotiation(socket, videoElementId) {
  // Create an RTCPeerConnection
  peerConnection = new RTCPeerConnection(null);
  
  // Create a MediaStream
  peerStream = new MediaStream();
  
  // Set the video element source
  const video = document.getElementById(videoElementId);
  video.srcObject = peerStream;
  video.onplay = function() {
    video.muted = false;
  };
  
  // Handle the audio/video stream
  peerConnection.ontrack = function(event) {
    console.log('Video stream received: ' + event.streams.length + ' stream(s)');
    peerStream.addTrack(event.track);
  };
  
  // Add audio and video receivers
  peerConnection.addTransceiver('audio', { direction: 'recvonly' });
  peerConnection.addTransceiver('video', { direction: 'recvonly' });
  
  // Create and send the SDP
  peerConnection.createOffer().then(offer => {
    return peerConnection.setLocalDescription(offer);
  }).then(() => {
    const message = {
      type: 'playStream',
      data: {
        sdp: peerConnection.localDescription.sdp
      }
    };
    socket.send(JSON.stringify(message));
    console.log('WebRTC negotiation request sent');
  }).catch(error => {
    console.error('WebRTC negotiation error: ' + error);
  });
}

// Handle the SDP answer
function handleSdpAnswer(sdp) {
  if (peerConnection) {
    const remoteDescription = new RTCSessionDescription({ type: 'answer', sdp: sdp });
    peerConnection.setRemoteDescription(remoteDescription).then(() => {
      console.log('WebRTC negotiation succeeded');
    }).catch(error => {
      console.error('WebRTC negotiation-answer error: ' + error);
    });
  }
}

// Close the WebRTC connection
function disconnectWebRTC() {
  if (peerConnection) {
    peerConnection.close();
    peerConnection = null;
  }
  if (peerStream) {
    peerStream.getTracks().forEach(track => track.stop());
    peerStream = null;
  }
  console.log('WebRTC connection closed');
}

// Example: add this to the WebSocket message handler
function handleWebSocketMessage(message, socket) {
  switch (message.type) {
    case 'avator-connect':
      if (message.code === 200) {
        console.log('Authentication succeeded; session ID: ' + message.data.sessionid);
        // Start media negotiation
        startMediaNegotiation(socket, 'remote-video');
      } else {
        console.log('Authentication failed: ' + message.msg);
      }
      break;
    case 'avator-play':
      if (message.code === 200 && message.data && message.data.sdp) {
        // Handle the SDP answer
        handleSdpAnswer(message.data.sdp);
      }
      break;
    // Handle other messages...
  }
}

// RTMP player example (using video.js)
// First, include video.js and the videojs-flash plugin
// <script src="https://cdn.jsdelivr.net/npm/video.js@7.15.4/dist/video.min.js"></script>
// <script src="https://cdn.jsdelivr.net/npm/videojs-flash@2.2.1/dist/videojs-flash.min.js"></script>

// Initialize the player
const player = videojs('remote-video', {
  techOrder: ['html5', 'flash'],
  autoplay: true,
  controls: true,
  sources: [
    {
      src: 'rtmp://your-rtmp-server/live/stream',
      type: 'rtmp/flv'
    }
  ]
});

// Listen for player events
player.on('play', function() {
  console.log('Player started playback');
});

player.on('error', function() {
  console.error('Player error:', player.error());
});

player.on('ended', function() {
  console.log('Playback ended');
});

// Send a playRtmp request
function sendPlayRtmp(ws, rtmpUrl) {
  const playRtmpMessage = {
    type: 'playRtmp',
    data: {
      url: rtmpUrl
    }
  };
  
  ws.send(JSON.stringify(playRtmpMessage));
  console.log('playRtmp request sent:', playRtmpMessage);
}

// Handle the avator-play response
function handleAvatorPlay(message) {
  if (message.code === 200) {
    console.log('RTMP media negotiation succeeded; starting playback');
    // The configured player starts playback automatically
  } else {
    console.error('RTMP media negotiation failed:', message.msg);
  }
}