Class ComputerUseCallManager

Manages and executes computer actions that can be called by AI agents.

Inheritance

object

Object

Component

Behaviour

MonoBehaviour

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>

ComputerUseCallManager

Implements

IComputerUseExecutor

IToolCallExecutor<ComputerUseCall, ComputerUseOutput>

IComputerUseActionHandler

Inherited Members

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.ValidateCall(ComputerUseCall)

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.ExecuteAsync(ComputerUseCall, CancellationToken)

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.ExecuteAsyncInternal(ComputerUseCall, CancellationToken)

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.CreateErrorOutput(ComputerUseCall, Exception)

MonoBehaviour.IsInvoking()

MonoBehaviour.CancelInvoke()

MonoBehaviour.Invoke(string, float)

MonoBehaviour.InvokeRepeating(string, float, float)

MonoBehaviour.CancelInvoke(string)

MonoBehaviour.IsInvoking(string)

MonoBehaviour.StartCoroutine(string)

MonoBehaviour.StartCoroutine(string, object)

MonoBehaviour.StartCoroutine(IEnumerator)

MonoBehaviour.StartCoroutine_Auto(IEnumerator)

MonoBehaviour.StopCoroutine(IEnumerator)

MonoBehaviour.StopCoroutine(Coroutine)

MonoBehaviour.StopCoroutine(string)

MonoBehaviour.StopAllCoroutines()

MonoBehaviour.print(object)

MonoBehaviour.destroyCancellationToken

MonoBehaviour.useGUILayout

MonoBehaviour.didStart

MonoBehaviour.didAwake

MonoBehaviour.runInEditMode

Behaviour.enabled

Behaviour.isActiveAndEnabled

Component.GetComponent(Type)

Component.GetComponent<T>()

Component.TryGetComponent(Type, out Component)

Component.TryGetComponent<T>(out T)

Component.GetComponent(string)

Component.GetComponentInChildren(Type, bool)

Component.GetComponentInChildren(Type)

Component.GetComponentInChildren<T>(bool)

Component.GetComponentInChildren<T>()

Component.GetComponentsInChildren(Type, bool)

Component.GetComponentsInChildren(Type)

Component.GetComponentsInChildren<T>(bool)

Component.GetComponentsInChildren<T>(bool, List<T>)

Component.GetComponentsInChildren<T>()

Component.GetComponentsInChildren<T>(List<T>)

Component.GetComponentInParent(Type, bool)

Component.GetComponentInParent(Type)

Component.GetComponentInParent<T>(bool)

Component.GetComponentInParent<T>()

Component.GetComponentsInParent(Type, bool)

Component.GetComponentsInParent(Type)

Component.GetComponentsInParent<T>(bool)

Component.GetComponentsInParent<T>(bool, List<T>)

Component.GetComponentsInParent<T>()

Component.GetComponents(Type)

Component.GetComponents(Type, List<Component>)

Component.GetComponents<T>(List<T>)

Component.GetComponents<T>()

Component.GetComponentIndex()

Component.CompareTag(string)

Component.CompareTag(TagHandle)

Component.SendMessageUpwards(string, object, SendMessageOptions)

Component.SendMessageUpwards(string, object)

Component.SendMessageUpwards(string)

Component.SendMessageUpwards(string, SendMessageOptions)

Component.SendMessage(string, object)

Component.SendMessage(string)

Component.SendMessage(string, object, SendMessageOptions)

Component.SendMessage(string, SendMessageOptions)

Component.BroadcastMessage(string, object, SendMessageOptions)

Component.BroadcastMessage(string, object)

Component.BroadcastMessage(string)

Component.BroadcastMessage(string, SendMessageOptions)

Component.transform

Component.transformHandle

Component.gameObject

Component.tag

Object.GetEntityId()

Object.GetInstanceID()

Object.GetHashCode()

Object.Equals(object)

Object.InstantiateAsync<T>(T)

Object.InstantiateAsync<T>(T, Transform)

Object.InstantiateAsync<T>(T, Vector3, Quaternion)

Object.InstantiateAsync<T>(T, Transform, Vector3, Quaternion)

Object.InstantiateAsync<T>(T, int)

Object.InstantiateAsync<T>(T, int, Transform)

Object.InstantiateAsync<T>(T, int, Vector3, Quaternion)

Object.InstantiateAsync<T>(T, int, ReadOnlySpan<Vector3>, ReadOnlySpan<Quaternion>)

Object.InstantiateAsync<T>(T, int, Transform, Vector3, Quaternion)

Object.InstantiateAsync<T>(T, int, Transform, Vector3, Quaternion, CancellationToken)

Object.InstantiateAsync<T>(T, int, Transform, ReadOnlySpan<Vector3>, ReadOnlySpan<Quaternion>)

Object.InstantiateAsync<T>(T, int, Transform, ReadOnlySpan<Vector3>, ReadOnlySpan<Quaternion>, CancellationToken)

Object.InstantiateAsync<T>(T, InstantiateParameters, CancellationToken)

Object.InstantiateAsync<T>(T, int, InstantiateParameters, CancellationToken)

Object.InstantiateAsync<T>(T, Vector3, Quaternion, InstantiateParameters, CancellationToken)

Object.InstantiateAsync<T>(T, int, Vector3, Quaternion, InstantiateParameters, CancellationToken)

Object.InstantiateAsync<T>(T, int, ReadOnlySpan<Vector3>, ReadOnlySpan<Quaternion>, InstantiateParameters, CancellationToken)

Object.Instantiate(Object, Vector3, Quaternion)

Object.Instantiate(Object, Vector3, Quaternion, Transform)

Object.Instantiate(Object)

Object.Instantiate(Object, Scene)

Object.Instantiate<T>(T, InstantiateParameters)

Object.Instantiate<T>(T, Vector3, Quaternion, InstantiateParameters)

Object.Instantiate(Object, Transform)

Object.Instantiate(Object, Transform, bool)

Object.Instantiate<T>(T)

Object.Instantiate<T>(T, Vector3, Quaternion)

Object.Instantiate<T>(T, Vector3, Quaternion, Transform)

Object.Instantiate<T>(T, Transform)

Object.Instantiate<T>(T, Transform, bool)

Object.Destroy(Object, float)

Object.Destroy(Object)

Object.DestroyImmediate(Object, bool)

Object.DestroyImmediate(Object)

Object.FindObjectsOfType(Type)

Object.FindObjectsOfType(Type, bool)

Object.FindObjectsByType(Type, FindObjectsSortMode)

Object.FindObjectsByType(Type, FindObjectsInactive, FindObjectsSortMode)

Object.DontDestroyOnLoad(Object)

Object.DestroyObject(Object, float)

Object.DestroyObject(Object)

Object.FindSceneObjectsOfType(Type)

Object.FindObjectsOfTypeIncludingAssets(Type)

Object.FindObjectsOfType<T>()

Object.FindObjectsByType<T>(FindObjectsSortMode)

Object.FindObjectsOfType<T>(bool)

Object.FindObjectsByType<T>(FindObjectsInactive, FindObjectsSortMode)

Object.FindObjectOfType<T>()

Object.FindObjectOfType<T>(bool)

Object.FindFirstObjectByType<T>()

Object.FindAnyObjectByType<T>()

Object.FindFirstObjectByType<T>(FindObjectsInactive)

Object.FindAnyObjectByType<T>(FindObjectsInactive)

Object.FindObjectsOfTypeAll(Type)

Object.FindObjectOfType(Type)

Object.FindFirstObjectByType(Type)

Object.FindAnyObjectByType(Type)

Object.FindObjectOfType(Type, bool)

Object.FindFirstObjectByType(Type, FindObjectsInactive)

Object.FindAnyObjectByType(Type, FindObjectsInactive)

Object.ToString()

Object.name

Object.hideFlags

object.Equals(object, object)

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

Namespace: Glitch9.AIDevKit.Agents

Assembly: Glitch9.AIDevKit.Agent.dll

Syntax

[AddComponentMenu("AI DevKit/AI Agents/Computer Use Call Manager", 21)]
public class ComputerUseCallManager : ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>, IComputerUseExecutor, IToolCallExecutor<ComputerUseCall, ComputerUseOutput>, IComputerUseActionHandler

Fields

onInputEventReceived

Declaration

public UnityEvent<Event> onInputEventReceived

Field Value

Type	Description
UnityEvent<Event>

onKeyboardTyped

Declaration

public UnityEvent<string> onKeyboardTyped

Field Value

Type	Description
UnityEvent<string>

Properties

Api

Declaration

public Api Api { get; set; }

Property Value

Type	Description
Api

ScreenshotPathType

Declaration

public ScreenshotPathType ScreenshotPathType { get; }

Property Value

Type	Description
ScreenshotPathType

Methods

CanExecute(string)

Always returns true; this manager handles all computer use calls regardless of name.

Declaration

public bool CanExecute(string toolName)

Parameters

Type	Name	Description
string	toolName

Returns

Type	Description
bool

CreateErrorOutput(ComputerUseCall, Exception)

Declaration

protected override ComputerUseOutput CreateErrorOutput(ComputerUseCall call, Exception ex)

Parameters

Type	Name	Description
ComputerUseCall	call
Exception	ex

Returns

Type	Description
ComputerUseOutput

Overrides

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.CreateErrorOutput(ComputerUseCall, Exception)

ExecuteAsyncInternal(ComputerUseCall, CancellationToken)

Executes the given ComputerUseCall by dispatching its action to the platform handler.

Declaration

protected override UniTask<ComputerUseOutput> ExecuteAsyncInternal(ComputerUseCall call, CancellationToken ct)

Parameters

Type	Name	Description
ComputerUseCall	call	The computer use call to execute.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask<ComputerUseOutput>	The output produced by the executed computer use action, or null on failure.

Overrides

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.ExecuteAsyncInternal(ComputerUseCall, CancellationToken)

OnKeyboardKeysPressedAsync(List<string>, CancellationToken)

Simulates pressing a sequence of keyboard keys (e.g., "Ctrl", "Shift", "P").

Declaration

public UniTask OnKeyboardKeysPressedAsync(List<string> keys, CancellationToken ct)

Parameters

Type	Name	Description
List<string>	keys	List of key names to press in order.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnKeyboardTypedAsync(string, CancellationToken)

Simulates typing the given text string as keyboard input.

Declaration

public UniTask OnKeyboardTypedAsync(string text, CancellationToken ct)

Parameters

Type	Name	Description
string	text	The text to type.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnMouseClickAsync(MouseButton, Vector2, CancellationToken)

Simulates a mouse button click at the specified screen position.

Declaration

public UniTask OnMouseClickAsync(MouseButton button, Vector2 position, CancellationToken ct)

Parameters

Type	Name	Description
MouseButton	button	The mouse button to click.
Vector2	position	The screen position of the click.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnMouseDoubleClickAsync(Vector2, CancellationToken)

Simulates a mouse double-click at the specified screen position.

Declaration

public UniTask OnMouseDoubleClickAsync(Vector2 position, CancellationToken ct)

Parameters

Type	Name	Description
Vector2	position	The screen position of the double-click.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnMouseDragAsync(List<Vector2>, CancellationToken)

Simulates a mouse drag operation along the specified path of screen positions.

Declaration

public UniTask OnMouseDragAsync(List<Vector2> path, CancellationToken ct)

Parameters

Type	Name	Description
List<Vector2>	path	Ordered list of screen positions to drag through.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnMouseMoveAsync(Vector2, CancellationToken)

Moves the mouse cursor to the specified screen position.

Declaration

public UniTask OnMouseMoveAsync(Vector2 position, CancellationToken ct)

Parameters

Type	Name	Description
Vector2	position	The target screen position.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnMouseScrollAsync(Vector2, Vector2, CancellationToken)

Simulates a mouse scroll wheel event at the specified screen position.

Declaration

public UniTask OnMouseScrollAsync(Vector2 position, Vector2 scroll, CancellationToken ct)

Parameters

Type	Name	Description
Vector2	position	The screen position of the scroll event.
Vector2	scroll	The scroll delta in X and Y axes.
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

OnTakeScreenshotAsync(CancellationToken)

Takes a screenshot and returns either an uploaded file ID or a cloud storage URL depending on ScreenshotPathType.

Declaration

public UniTask<string> OnTakeScreenshotAsync(CancellationToken ct)

Parameters

Type	Name	Description
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask<string>	The uploaded file ID if FileId; otherwise a URL string.

OnWaitAsync(CancellationToken)

Waits for a platform-defined duration before the next action is executed.

Declaration

public UniTask OnWaitAsync(CancellationToken ct)

Parameters

Type	Name	Description
CancellationToken	ct	Cancellation token.

Returns

Type	Description
UniTask

SetSafetyCheckService(IComputerUseSafetyCheckService)

Sets the safety check service used to approve or reject pending computer use safety checks.

Declaration

public void SetSafetyCheckService(IComputerUseSafetyCheckService service)

Parameters

Type	Name	Description
IComputerUseSafetyCheckService	service	The safety check service implementation.

ValidateCall(ComputerUseCall)

Declaration

protected override void ValidateCall(ComputerUseCall call)

Parameters

Type	Name	Description
ComputerUseCall	call

Overrides

ToolCallManagerBase<ComputerUseCall, ComputerUseOutput>.ValidateCall(ComputerUseCall)

Implements

IComputerUseExecutor

IToolCallExecutor<TCall, TOutput>

IComputerUseActionHandler

Extension Methods

LocalizationExtensions.SetLocalizedAsset<TComponent, TObject>(TComponent, string, Action<TComponent, TObject>, string, LocalizationOptions<TObject>)

ArrayExtensions.ToArrayOrEmpty<T>(T)

ArrayExtensions.ToArrayOrNull<T>(T)

EventExtensions.ToDelta<T>(T, string, int, string, bool)

EventExtensions.ToDone<T>(T, string, int, string)

EventExtensions.ToEvent<T>(T)

EventExtensions.UpcastDelta<T1, T2>(T1, string, int, string, bool)

FallbackExtensions.IsOr<TParent, TChild>(TParent, TChild)

FileExtensions.ToFile<T>(T, string)

ResponseCastingExtensions.GetResult<T>(T)

UnityObjectExtensions.Find<T>(T, string)

UnityObjectExtensions.Find<T, TTarget>(TTarget, T, string)

UnityObjectExtensions.Find<T, TTarget>(TTarget, T, string, string)

UnityObjectExtensions.GetChildIndex<T>(T)

UnityObjectExtensions.GetOrAddComponentInChildren<T, TV>(TV, string)

SystemExtensions.GetName(object)

UnityObjectExtensions.EncodeToBase64(Object)

UnityObjectExtensions.ToBytes(Object)