VoxtaClient.h

Go to the documentation of this file.

// Copyright(c) 2024 grrimgrriefer & DZnnah, see LICENSE for details.
 
#pragma once
#include "CoreMinimal.h"
#include "Subsystems/GameInstanceSubsystem.h"
#include "VoxtaData/Public/VoxtaClientState.h"
#include "VoxtaDefines.h"
#include "UserCharData.h"
#include "VoxtaClient.generated.h"
 
class FSignalRValue;
class IHubConnection;
class FSignalRInvokeResult;
class UVoxtaAudioInput;
class Audio2FaceRESTHandler;
class UVoxtaAudioPlayback;
class AVoxtaGlobalAudioPlaybackHolder;
class VoxtaLogger;
class VoxtaApiRequestHandler;
class VoxtaApiResponseHandler;
class TexturesCacheHandler;
struct ServerResponseBase;
struct ServerResponseError;
struct ServerResponseChatMessageBase;
struct ServerResponseWelcome;
struct ServerResponseCharacterList;
struct ServerResponseChatStarted;
struct ServerResponseChatUpdate;
struct ServerResponseSpeechTranscription;
struct ServerResponseContextUpdated;
struct ServerResponseChatClosed;
struct ServerResponseChatSessionError;
struct ServerResponseConfiguration;
struct FAiCharData;
struct FBaseCharData;
struct FChatSession;
struct FChatMessage;
struct FVoxtaVersionData;

UCLASS(DisplayName = "Voxta Client", Category = "Voxta")
class UNREALVOXTA_API UVoxtaClient : public UGameInstanceSubsystem
{
    GENERATED_BODY()
 
#pragma region delegate declarations
public:
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientStateChanged, VoxtaClientState, newState);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientCharacterRegistered, const FAiCharData&, charData);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FVoxtaClientCharMessageAdded, const FBaseCharData&, sender, const FChatMessage&, message);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientCharMessageRemoved, const FChatMessage&, message);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientSpeechTranscribed, const FString&, message);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatSessionStarted, const FChatSession&, chatSession);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatSessionStopped, const FChatSession&, chatSession);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FVoxtaClientAudioPlaybackRegistered, const UVoxtaAudioPlayback*, playbackHandler, const FGuid&, characterId);
    DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatContextUpdated, const FString&, newContext);

    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientStateChangedNative, VoxtaClientState);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientCharacterRegisteredNative, const FAiCharData&);
    DECLARE_MULTICAST_DELEGATE_TwoParams(FVoxtaClientCharMessageAddedNative, const FBaseCharData&, const FChatMessage&);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientCharMessageRemovedNative, const FChatMessage&);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientSpeechTranscribedNative, const FString&);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatSessionStartedNative, const FChatSession&);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatSessionStoppedNative, const FChatSession&);
    DECLARE_MULTICAST_DELEGATE_TwoParams(FVoxtaClientAudioPlaybackRegisteredNative, const UVoxtaAudioPlayback*, const FGuid&);
    DECLARE_MULTICAST_DELEGATE_OneParam(FVoxtaClientChatContextUpdatedNative, const FString&);
#pragma endregion
 
#pragma region events
public:
    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientStateChanged VoxtaClientStateChangedEvent;
    FVoxtaClientStateChangedNative VoxtaClientStateChangedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientCharacterRegistered VoxtaClientCharacterRegisteredEvent;
    FVoxtaClientCharacterRegisteredNative VoxtaClientCharacterRegisteredEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientCharMessageAdded VoxtaClientCharMessageAddedEvent;
    FVoxtaClientCharMessageAddedNative VoxtaClientCharMessageAddedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientCharMessageRemoved VoxtaClientCharMessageRemovedEvent;
    FVoxtaClientCharMessageRemovedNative VoxtaClientCharMessageRemovedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientSpeechTranscribed VoxtaClientSpeechTranscribedPartialEvent;
    FVoxtaClientSpeechTranscribedNative VoxtaClientSpeechTranscribedPartialEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientSpeechTranscribed VoxtaClientSpeechTranscribedCompleteEvent;
    FVoxtaClientSpeechTranscribedNative VoxtaClientSpeechTranscribedCompleteEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientChatContextUpdated VoxtaClientChatContextUpdatedEvent;
    FVoxtaClientChatContextUpdatedNative VoxtaClientChatContextUpdatedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientChatSessionStarted VoxtaClientChatSessionStartedEvent;
    FVoxtaClientChatSessionStartedNative VoxtaClientChatSessionStartedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientChatSessionStopped VoxtaClientChatSessionStoppedEvent;
    FVoxtaClientChatSessionStoppedNative VoxtaClientChatSessionStoppedEventNative;

    UPROPERTY(BlueprintAssignable, Category = "Voxta", meta = (IsBindableEvent = "True"))
    FVoxtaClientAudioPlaybackRegistered VoxtaClientAudioPlaybackRegisteredEvent;
    FVoxtaClientAudioPlaybackRegisteredNative VoxtaClientAudioPlaybackRegisteredEventNative;
#pragma endregion
 
#pragma region UGameInstanceSubsystem overrides
public:
    virtual void Initialize(FSubsystemCollectionBase& collection) override;
    virtual void Deinitialize() override;
#pragma endregion
 
#pragma region public API
public:
    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void StartConnection(const FString& ipv4Address, int port);

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void Disconnect(bool silent = false);

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void StartChatWithCharacter(const FGuid& charId, const FString& context = TEXT(""));

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void SetGlobalAudioFallbackEnabled(bool newState);

    UFUNCTION(BlueprintPure, Category = "Voxta")
    bool IsGlobalAudioFallbackActive() const;

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void StopActiveChat();

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void UpdateChatContext(const FString& newContext);

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void SendUserInput(const FString& inputText, bool generateReply = true, bool characterActionInference = false);

    UFUNCTION(BlueprintPure, Category = "Voxta")
    const FString& GetServerAddress() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    int GetServerPort() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    UVoxtaAudioInput* GetVoiceInputHandler() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    VoxtaClientState GetCurrentState() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FGuid GetUserId() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FString GetBrowserUrlForCharacter(const FGuid& aiCharacterId) const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FGuid GetMainAssistantId() const;

    void TryFetchAndCacheCharacterThumbnail(const FGuid& baseCharacterId, FDownloadedTextureDelegateNative onThumbnailFetched);

    UFUNCTION(BlueprintPure, Category = "Voxta")
    const UVoxtaAudioPlayback* GetRegisteredAudioPlaybackHandlerForID(const FGuid& characterId) const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FAiCharData GetAiCharacterDataCopyById(const FGuid& characterId) const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    TArray<FAiCharData> GetAvailableAiCharactersCopy() const;

    bool TryRegisterPlaybackHandler(const FGuid& characterId, TWeakObjectPtr<UVoxtaAudioPlayback> playbackHandler);

    bool TryUnregisterPlaybackHandler(const FGuid& characterId);

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FChatSession GetChatSessionCopy() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    FVoxtaVersionData GetServerVersionCopy() const;

    UFUNCTION(BlueprintPure, Category = "Voxta")
    bool IsMatchingAPIVersion() const;

    const FChatSession* GetChatSession() const;

    TWeakPtr<Audio2FaceRESTHandler> GetA2FHandler() const;

    UFUNCTION(BlueprintCallable, Category = "Voxta")
    void SetCensoredLogs(bool isCensorActive);

    UFUNCTION(BlueprintPure, Category = "Voxta")
    bool IsLogCensorActive() const;
#pragma endregion
 
#pragma region data
private:
    const FString SEND_MESSAGE_EVENT_NAME = TEXT("SendMessage");
    const FString RECEIVE_MESSAGE_EVENT_NAME = TEXT("ReceiveMessage");
 
    UPROPERTY()
    UVoxtaAudioInput* m_voiceInput;
 
    UPROPERTY()
    AVoxtaGlobalAudioPlaybackHolder* m_globalAudioPlaybackComp;
    FDelegateHandle globalAudioPlaybackHandle;
    bool m_enableGlobalAudioFallback = true;
 
    TSharedPtr<VoxtaLogger> m_logUtility;
    TSharedPtr<IHubConnection> m_hub;
    TSharedPtr<Audio2FaceRESTHandler> m_A2FHandler;
    TSharedPtr<TexturesCacheHandler> m_texturesCacheHandler;
 
    VoxtaClientState m_currentState = VoxtaClientState::Disconnected;
    TUniquePtr<FUserCharData> m_userData;
    FGuid m_mainAssistantId;
    FString m_hostAddress;
    uint16 m_hostPort;
 
    TUniquePtr<FVoxtaVersionData> m_voxtaVersionData;
    TUniquePtr<FChatSession> m_chatSession;
    TArray<TUniquePtr<const FAiCharData>> m_characterList;
    TMap<FGuid, TWeakObjectPtr<UVoxtaAudioPlayback>> m_registeredCharacterAudioPlaybackComps;
    TMap<FGuid, FDelegateHandle> m_audioPlaybackHandles;
#pragma endregion
 
#pragma region private API
private:
    void StartListeningToServer();
 
#pragma region IHubConnection listeners
private:
    void OnReceivedMessage(const TArray<FSignalRValue>& arguments);
    void OnConnected();
    void OnConnectionError(const FString& error);
    void OnClosed();
#pragma endregion

    void SendMessageToServer(const FSignalRValue& message);

    void OnMessageSent(const FSignalRInvokeResult& deliveryReceipt);

    template<typename T>
    bool HandleResponseHelper(const ServerResponseBase* response, const FString& logMessage,
        bool (UVoxtaClient::* handler)(const T&), bool isSensitive);
 
#pragma region VoxtaServer response handlers
private:
    bool HandleResponse(const TMap<FString, FSignalRValue>& responseData);
    bool HandleWelcomeResponse(const ServerResponseWelcome& response);
    bool HandleCharacterListResponse(const ServerResponseCharacterList& response);
    bool HandleChatStartedResponse(const ServerResponseChatStarted& response);
    bool HandleChatMessageResponse(const ServerResponseChatMessageBase& response);
    bool HandleChatUpdateResponse(const ServerResponseChatUpdate& response);
    bool HandleSpeechTranscriptionResponse(const ServerResponseSpeechTranscription& response);
    bool HandleErrorResponse(const ServerResponseError& response);
    bool HandleContextUpdateResponse(const ServerResponseContextUpdated& response);
    bool HandleChatClosedResponse(const ServerResponseChatClosed& response);
    bool HandleChatSessionErrorResponse(const ServerResponseChatSessionError& response);
    bool HandleConfigurationResponse(const ServerResponseConfiguration& response);
#pragma endregion
 
    void StopChatInternal();

    void NotifyAudioPlaybackComplete(const FGuid& messageId);

    void SetState(VoxtaClientState newState);

    const TUniquePtr<const FAiCharData>* GetAiCharacterDataById(const FGuid& charId) const;
 
    AVoxtaGlobalAudioPlaybackHolder* GetOrCreateGlobalAudioFallbackInternal();
#pragma endregion
};