ComputerUse

ComputerUseService

Properties:

service AddonProcessesService
mouse MouseService
keyboard KeyboardService
screen ScreenService
display DisplayService
a11y A11yService
recording RecordingService

Constructors

Constructor

new ComputerUseService(transport: ComputerUseTransport): ComputerUseService;

Parameters:

transport ComputerUseTransport

Returns:

ComputerUseService

Methods

restart()

restart(): Promise<ComputerUseRestartResponse>;

Returns:

Promise<ComputerUseRestartResponse>

start()

start(): Promise<ComputerUseLifecycleResponse>;

Returns:

Promise<ComputerUseLifecycleResponse>

status()

status(): Promise<ComputerUseStatusResponse>;

Returns:

Promise<ComputerUseStatusResponse>

stop()

stop(): Promise<ComputerUseLifecycleResponse>;

Returns:

Promise<ComputerUseLifecycleResponse>

AddonProcessesService

Constructors

Constructor

new AddonProcessesService(transport: ComputerUseTransport): AddonProcessesService;

Parameters:

transport ComputerUseTransport

Returns:

AddonProcessesService

Methods

errors()

errors(name: string): Promise<ComputerUseProcessErrorsResponse>;

Parameters:

name string

Returns:

Promise<ComputerUseProcessErrorsResponse>

logs()

logs(name: string): Promise<ComputerUseProcessLogsResponse>;

Parameters:

name string

Returns:

Promise<ComputerUseProcessLogsResponse>

restart()

restart(name: string): Promise<ComputerUseProcessRestartResponse>;

Parameters:

name string

Returns:

Promise<ComputerUseProcessRestartResponse>

status()

status(): Promise<Record<string, ComputerUseProcessStatus>>;

Returns:

Promise<Record<string, ComputerUseProcessStatus>>

status()

status(name: string): Promise<ComputerUseProcessRunningStatus>;

Parameters:

name string

Returns:

Promise<ComputerUseProcessRunningStatus>

status()

status(
   name?: string): Promise<Record<string, ComputerUseProcessStatus> | ComputerUseProcessRunningStatus>;

Parameters:

name? string

Returns:

Promise<Record<string, ComputerUseProcessStatus> | ComputerUseProcessRunningStatus>

A11yService

Constructors

Constructor

new A11yService(transport: ComputerUseTransport): A11yService;

Parameters:

transport ComputerUseTransport

Returns:

A11yService

Methods

find()

find(request: ComputerUseA11yFindRequest): Promise<ComputerUseA11yFindResponse>;

Parameters:

request ComputerUseA11yFindRequest

Returns:

Promise<ComputerUseA11yFindResponse>

focus()

focus(request: ComputerUseA11yFocusRequest): Promise<void>;

Parameters:

request ComputerUseA11yFocusRequest

Returns:

Promise<void>

invoke()

invoke(request: ComputerUseA11yInvokeRequest): Promise<void>;

Parameters:

request ComputerUseA11yInvokeRequest

Returns:

Promise<void>

setValue()

setValue(request: ComputerUseA11ySetValueRequest): Promise<void>;

Parameters:

request ComputerUseA11ySetValueRequest

Returns:

Promise<void>

tree()

tree(options: ComputerUseA11yTreeOptions): Promise<ComputerUseA11yTreeResponse>;

Parameters:

options ComputerUseA11yTreeOptions

Returns:

Promise<ComputerUseA11yTreeResponse>

DisplayService

Constructors

Constructor

new DisplayService(transport: ComputerUseTransport): DisplayService;

Parameters:

transport ComputerUseTransport

Returns:

DisplayService

Methods

info()

info(): Promise<ComputerUseDisplayInfoResponse>;

Returns:

Promise<ComputerUseDisplayInfoResponse>

windows()

windows(): Promise<ComputerUseDisplayWindowsResponse>;

Returns:

Promise<ComputerUseDisplayWindowsResponse>

KeyboardService

Constructors

Constructor

new KeyboardService(transport: ComputerUseTransport): KeyboardService;

Parameters:

transport ComputerUseTransport

Returns:

KeyboardService

Methods

hotkey()

hotkey(request: ComputerUseKeyboardHotkeyRequest): Promise<void>;

Parameters:

request ComputerUseKeyboardHotkeyRequest

Returns:

Promise<void>

key()

key(request: ComputerUseKeyboardKeyRequest): Promise<void>;

Parameters:

request ComputerUseKeyboardKeyRequest

Returns:

Promise<void>

type()

type(request: ComputerUseKeyboardTypeRequest): Promise<void>;

Parameters:

request ComputerUseKeyboardTypeRequest

Returns:

Promise<void>

MouseService

Constructors

Constructor

new MouseService(transport: ComputerUseTransport): MouseService;

Parameters:

transport ComputerUseTransport

Returns:

MouseService

Methods

click()

click(request: ComputerUseMouseClickRequest): Promise<ComputerUsePoint>;

Parameters:

request ComputerUseMouseClickRequest

Returns:

Promise<ComputerUsePoint>

drag()

drag(request: ComputerUseMouseDragRequest): Promise<ComputerUsePoint>;

Parameters:

request ComputerUseMouseDragRequest

Returns:

Promise<ComputerUsePoint>

move()

move(request: ComputerUseMouseMoveRequest): Promise<ComputerUsePoint>;

Parameters:

request ComputerUseMouseMoveRequest

Returns:

Promise<ComputerUsePoint>

position()

position(): Promise<ComputerUsePoint>;

Returns:

Promise<ComputerUsePoint>

scroll()

scroll(request: ComputerUseMouseScrollRequest): Promise<ComputerUseScrollResponse>;

Parameters:

request ComputerUseMouseScrollRequest

Returns:

Promise<ComputerUseScrollResponse>

RecordingService

Constructors

Constructor

new RecordingService(transport: ComputerUseTransport): RecordingService;

Parameters:

transport ComputerUseTransport

Returns:

RecordingService

Methods

delete()

delete(id: string): Promise<void>;

Parameters:

id string

Returns:

Promise<void>

download()

download(id: string): Promise<Uint8Array>;

Parameters:

id string

Returns:

Promise<Uint8Array>

get()

get(id: string): Promise<ComputerUseRecording>;

Parameters:

id string

Returns:

Promise<ComputerUseRecording>

list()

list(): Promise<ComputerUseRecordingsResponse>;

Returns:

Promise<ComputerUseRecordingsResponse>

start()

start(request: ComputerUseRecordingStartRequest = {}): Promise<ComputerUseRecording>;

Parameters:

request ComputerUseRecordingStartRequest = {}

Returns:

Promise<ComputerUseRecording>

stop()

stop(request: ComputerUseRecordingStopRequest): Promise<ComputerUseRecording>;

Parameters:

request ComputerUseRecordingStopRequest

Returns:

Promise<ComputerUseRecording>

ScreenService

Constructors

Constructor

new ScreenService(transport: ComputerUseTransport): ScreenService;

Parameters:

transport ComputerUseTransport

Returns:

ScreenService

Methods

capture()

capture(
   options: { readonly showCursor?: boolean } = {}): Promise<ComputerUseScreenshotResponse>;

Parameters:

options { readonly showCursor?: boolean } = {}

Returns:

Promise<ComputerUseScreenshotResponse>

captureCompressed()

captureCompressed(
   options: ComputerUseScreenshotCompressedOptions = {}): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotCompressedOptions = {}

Returns:

Promise<ComputerUseScreenshotResponse>

region()

region(
   options: ComputerUseScreenshotRegionOptions): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotRegionOptions

Returns:

Promise<ComputerUseScreenshotResponse>

regionCompressed()

regionCompressed(
   options: ComputerUseScreenshotRegionCompressedOptions): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotRegionCompressedOptions

Returns:

Promise<ComputerUseScreenshotResponse>

screenshot()

screenshot(
   options: { readonly showCursor?: boolean } = {}): Promise<ComputerUseScreenshotResponse>;

Parameters:

options { readonly showCursor?: boolean } = {}

Returns:

Promise<ComputerUseScreenshotResponse>

screenshotCompressed()

screenshotCompressed(
   options: ComputerUseScreenshotCompressedOptions = {}): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotCompressedOptions = {}

Returns:

Promise<ComputerUseScreenshotResponse>

screenshotRegion()

screenshotRegion(
   options: ComputerUseScreenshotRegionOptions): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotRegionOptions

Returns:

Promise<ComputerUseScreenshotResponse>

screenshotRegionCompressed()

screenshotRegionCompressed(
   options: ComputerUseScreenshotRegionCompressedOptions): Promise<ComputerUseScreenshotResponse>;

Parameters:

options ComputerUseScreenshotRegionCompressedOptions

Returns:

Promise<ComputerUseScreenshotResponse>

ComputerUseTransport

Methods

delete()

delete<T>(path: string): Promise<T>;

Parameters:

path string

Returns:

Promise<T>

get()

get<T>(path: string, config?: { params?: Record<string, unknown> }): Promise<T>;

Parameters:

path string
config? { params?: Record<string, unknown> }

Returns:

Promise<T>

getBytes()

getBytes(path: string): Promise<ArrayBuffer>;

Parameters:

path string

Returns:

Promise<ArrayBuffer>

post()

post<T>(path: string, body?: unknown): Promise<T>;

Parameters:

path string
body? unknown

Returns:

Promise<T>

ComputerUseActivityStatus

type ComputerUseActivityStatus = 'active' | 'inactive' | 'unknown'

ComputerUseDesktopState

type ComputerUseDesktopState = 'not_started' | 'starting' | 'ready' | 'failed' | 'unknown'

ComputerUseDesktopFailureCode

type ComputerUseDesktopFailureCode =
  | 'desktop_exited'
  | 'addon_crashed'
  | 'start_failed'
  | 'start_interrupted'
  | 'session_ended'
  | 'session_replaced'
  | 'state_corrupt'
  | 'state_persistence_failed'
  | 'unknown'

ComputerUseDesktopFailure

Properties:

code ComputerUseDesktopFailureCode
rawCode string
message string
extraFields? Record<string, unknown>

ComputerUseFailedStatus

Properties:

status ‘inactive’
desktop ‘failed’
cause ComputerUseDesktopFailure
extraFields? Record<string, unknown>

ComputerUseNotStartedStatus

Properties:

status ‘inactive’
desktop ‘not_started’
cause? never
extraFields? Record<string, unknown>

ComputerUseReadyStatus

Properties:

status ‘active’
desktop ‘ready’
cause? never
extraFields? Record<string, unknown>

ComputerUseRestartResponse

Properties:

message string
status ‘active’
desktop ‘ready’
extraFields? Record<string, unknown>

ComputerUseStartingStatus

Properties:

status ‘inactive’
desktop ‘starting’
cause? never
extraFields? Record<string, unknown>

ComputerUseUnknownStatus

A (status, desktop) pair this SDK release does not recognize, because a newer daemon added a value or a combination. rawStatus / rawDesktop carry what the daemon sent; the recognized variants need no raw sibling, since their typed values are the wire values.

Properties:

status ‘unknown’
desktop ‘unknown’
rawStatus string
rawDesktop string
cause? ComputerUseDesktopFailure - The failure the daemon attached to the unrecognized state, when it sent one.
extraFields? Record<string, unknown>

ComputerUseA11yFindRequest

Properties:

scope ComputerUseA11yScope
pid? number
role? string
name? string
nameMatch? ComputerUseA11yNameMatch
states? string[]
limit? number

ComputerUseA11yFindResponse

Properties:

matches ComputerUseAccessibilityNode[]
truncated boolean

ComputerUseA11yFocusRequest

Properties:

id string

ComputerUseA11yInvokeRequest

Properties:

id string
action? string

ComputerUseA11yNameMatch

type ComputerUseA11yNameMatch = 'exact' | 'substring' | 'regex'

ComputerUseA11yScope

type ComputerUseA11yScope = 'focused' | 'pid' | 'all'

ComputerUseA11ySetValueRequest

Properties:

id string
value string

ComputerUseA11yTreeOptions

Properties:

scope ComputerUseA11yScope
pid? number
maxDepth? number

ComputerUseA11yTreeResponse

Properties:

root ComputerUseAccessibilityNode
truncated boolean

ComputerUseAccessibilityBounds

Properties:

x number
y number
width number
height number

ComputerUseAccessibilityNode

Properties:

id string
role string
name string
description? string
value? string - A control’s current value (macOS). Never set for password fields.
valueTruncated boolean - value was cut at the platform’s cap. false when uncut or not reported; the SDK always fills it.
bounds ComputerUseAccessibilityBounds
states string[]
actions? string[]
children? ComputerUseAccessibilityNode[]

ComputerUseDisplay

Properties:

id number
x number
y number
width number
height number
isActive boolean

ComputerUseDisplayInfoResponse

Properties:

displays ComputerUseDisplay[]

ComputerUseDisplayWindowsResponse

Properties:

windows ComputerUseWindow[]

ComputerUseKeyboardHotkeyRequest

Properties:

keys string

ComputerUseKeyboardKeyRequest

Properties:

key string
modifiers? ComputerUseKeyboardModifier[]

ComputerUseKeyboardModifier

type ComputerUseKeyboardModifier = 'ctrl' | 'alt' | 'shift' | 'cmd'

ComputerUseKeyboardTypeRequest

Properties:

text string
delay? number

ComputerUseLifecycleResponse

Properties:

message string
status Record<string, ComputerUseProcessStatus>

ComputerUseMouseButton

type ComputerUseMouseButton = 'left' | 'right' | 'middle'

ComputerUseMouseClickRequest

Properties:

x number
y number
button? ComputerUseMouseButton
double? boolean

ComputerUseMouseDragRequest

Properties:

startX number
startY number
endX number
endY number
button ComputerUseMouseButton

ComputerUseMouseMoveRequest

Properties:

x number
y number

ComputerUseMouseScrollRequest

Properties:

x number
y number
direction ComputerUseScrollDirection
amount? number

ComputerUsePoint

Properties:

x number
y number

ComputerUseProcessErrorsResponse

Properties:

processName string
errors string

ComputerUseProcessLogsResponse

Properties:

processName string
logs string

ComputerUseProcessRestartResponse

Properties:

message string
processName string

ComputerUseProcessRunningStatus

Properties:

processName string
running boolean

ComputerUseProcessStatus

Properties:

running? boolean
priority? number
autoRestart? boolean
pid? number
failed? boolean
restartAttempts? number
restartBackoffMs? number
status? string
extraFields? Record<string, unknown>

ComputerUseRecording

Properties:

id string
fileName string
filePath string
startTime string
endTime? string
status string
durationSeconds? number
sizeBytes? number

ComputerUseRecordingsResponse

Properties:

recordings ComputerUseRecording[]

ComputerUseRecordingStartRequest

Properties:

label? string

ComputerUseRecordingStopRequest

Properties:

id string

ComputerUseScreenshotCompressedOptions

Properties:

format? ComputerUseScreenshotFormat
quality? number
scale? number
showCursor? boolean

ComputerUseScreenshotFormat

type ComputerUseScreenshotFormat = 'png' | 'jpeg'

ComputerUseScreenshotRegionCompressedOptions

Properties:

x number
y number
width number
height number
showCursor? boolean
format? ComputerUseScreenshotFormat
quality? number
scale? number

ComputerUseScreenshotRegionOptions

Properties:

x number
y number
width number
height number
showCursor? boolean

ComputerUseScreenshotResponse

Properties:

screenshot string
cursorPosition? ComputerUsePoint
sizeBytes? number

ComputerUseScrollDirection

type ComputerUseScrollDirection = 'up' | 'down'

ComputerUseScrollResponse

Properties:

success boolean

ComputerUseStatusResponse

type ComputerUseStatusResponse =
  | ComputerUseNotStartedStatus
  | ComputerUseStartingStatus
  | ComputerUseReadyStatus
  | ComputerUseFailedStatus
  | ComputerUseUnknownStatus

ComputerUseWindow

Properties:

id number
title string
x number
y number
width number
height number
isActive boolean