Skip to content

Commit 4cda182

Browse files
authored
feat(electron-screen-capture): introduce new screen capture utilities (#937)
1 parent b500eb8 commit 4cda182

File tree

24 files changed

+851
-58
lines changed

24 files changed

+851
-58
lines changed

apps/stage-tamagotchi/electron.vite.config.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ export default defineConfig({
5252
'@proj-airi/stage-ui/*',
5353
'@proj-airi/drizzle-duckdb-wasm',
5454
'@proj-airi/drizzle-duckdb-wasm/*',
55+
'@proj-airi/electron-screen-capture',
5556

5657
// Static Assets: Models, Images, etc.
5758
'src/renderer/public/assets/*',

apps/stage-tamagotchi/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
"@proj-airi/audio": "workspace:^",
5454
"@proj-airi/ccc": "workspace:^",
5555
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
56+
"@proj-airi/electron-screen-capture": "workspace:^",
5657
"@proj-airi/font-cjkfonts-allseto": "workspace:^",
5758
"@proj-airi/font-xiaolai": "workspace:^",
5859
"@proj-airi/i18n": "workspace:^",
@@ -89,7 +90,6 @@
8990
"dompurify": "^3.3.1",
9091
"drizzle-kit": "^0.31.8",
9192
"drizzle-orm": "^0.45.1",
92-
"electron-audio-loopback": "^1.0.6",
9393
"electron-click-drag-plugin": "^2.0.2",
9494
"electron-updater": "^6.6.2",
9595
"es-toolkit": "^1.43.0",

apps/stage-tamagotchi/src/main/index.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ import { env, platform } from 'node:process'
22

33
import { electronApp, optimizer } from '@electron-toolkit/utils'
44
import { Format, LogLevel, setGlobalFormat, setGlobalLogLevel, useLogg } from '@guiiai/logg'
5+
import { initScreenCaptureForMain } from '@proj-airi/electron-screen-capture/main'
56
import { app, ipcMain } from 'electron'
6-
import { initMain as initAudioLoopback } from 'electron-audio-loopback'
77
import { noop } from 'es-toolkit'
88
import { createLoggLogger, injeca } from 'injeca'
99
import { isLinux } from 'std-env'
@@ -67,7 +67,7 @@ if (isLinux) {
6767
app.dock?.setIcon(icon)
6868
electronApp.setAppUserModelId('ai.moeru.airi')
6969

70-
initAudioLoopback()
70+
initScreenCaptureForMain()
7171

7272
app.whenReady().then(async () => {
7373
injeca.setLogger(createLoggLogger(useLogg('injeca').useGlobalConfig()))

apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { dirname, join, resolve } from 'node:path'
22
import { fileURLToPath } from 'node:url'
33

4+
import { initScreenCaptureForWindow } from '@proj-airi/electron-screen-capture/main'
45
import { BrowserWindow } from 'electron'
56

67
import { baseUrl, getElectronMainDirname, load } from '../../libs/electron/location'
@@ -15,5 +16,8 @@ export async function setupBeatSync() {
1516
})
1617

1718
await load(window, baseUrl(resolve(getElectronMainDirname(), '..', 'renderer'), 'beat-sync.html'))
19+
20+
initScreenCaptureForWindow(window)
21+
1822
return window
1923
}

apps/stage-tamagotchi/src/main/windows/main/index.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import clickDragPlugin from 'electron-click-drag-plugin'
1313
import { is } from '@electron-toolkit/utils'
1414
import { defineInvokeHandler } from '@moeru/eventa'
1515
import { createContext } from '@moeru/eventa/adapters/electron/main'
16+
import { initScreenCaptureForWindow } from '@proj-airi/electron-screen-capture/main'
1617
import { defu } from 'defu'
1718
import { BrowserWindow, ipcMain, shell } from 'electron'
1819
import { isLinux, isMacOS } from 'std-env'
@@ -175,5 +176,7 @@ export async function setupMainWindow(params: {
175176
})
176177
}
177178

179+
initScreenCaptureForWindow(window)
180+
178181
return window
179182
}

apps/stage-tamagotchi/src/main/windows/settings/index.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { WidgetsWindowManager } from '../widgets'
44

55
import { join, resolve } from 'node:path'
66

7+
import { initScreenCaptureForWindow } from '@proj-airi/electron-screen-capture/main'
78
import { BrowserWindow, shell } from 'electron'
89

910
import icon from '../../../../resources/icon.png?asset'
@@ -49,6 +50,8 @@ export function setupSettingsWindowReusableFunc(params: {
4950
devtoolsMarkdownStressWindow: params.devtoolsMarkdownStressWindow,
5051
})
5152

53+
initScreenCaptureForWindow(window)
54+
5255
return window
5356
}).getWindow
5457
}

apps/stage-tamagotchi/src/renderer/beat-sync.main.ts

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,13 @@ import {
1111
createContext,
1212
} from '@proj-airi/stage-shared/beat-sync'
1313

14-
const { ipcRenderer } = window.electron
15-
1614
const context = createContext()
1715

1816
const changeState = defineInvoke(context, beatSyncStateChangedInvokeEventa)
1917
const signalBeat = defineInvoke(context, beatSyncBeatSignaledInvokeEventa)
2018

21-
function enableLoopbackAudio() {
22-
// electron-audio-loopback currently registers this handler internally
23-
return ipcRenderer.invoke('enable-loopback-audio')
24-
}
25-
26-
function disableLoopbackAudio() {
27-
// electron-audio-loopback currently registers this handler internally
28-
return ipcRenderer.invoke('disable-loopback-audio')
29-
}
30-
3119
const detector = createBeatSyncDetector({
3220
env: StageEnvironment.Tamagotchi,
33-
enableLoopbackAudio,
34-
disableLoopbackAudio,
3521
})
3622

3723
detector.on('stateChange', state => changeState(state))
Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
<script setup lang="ts">
2+
import type { SerializableDesktopCapturerSource } from '@proj-airi/electron-screen-capture'
3+
4+
import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue'
5+
import { Button } from '@proj-airi/ui'
6+
import { onBeforeUnmount, onMounted, ref } from 'vue'
7+
8+
const sources = ref<ScreenCaptureSource[]>([])
9+
const isRefetching = ref(false)
10+
const activeStreams = ref<MediaStream[]>([])
11+
12+
interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
13+
appIconURL?: string
14+
thumbnailURL?: string
15+
}
16+
17+
const { getSources, selectWithSource } = useElectronScreenCapture(window.electron.ipcRenderer, {
18+
types: ['screen', 'window'],
19+
fetchWindowIcons: true,
20+
})
21+
22+
function toObjectUrl(bytes: Uint8Array, mime: string) {
23+
return URL.createObjectURL(new Blob([bytes.slice().buffer], { type: mime }))
24+
}
25+
26+
async function startCapture(source: SerializableDesktopCapturerSource) {
27+
try {
28+
await selectWithSource(() => source.id, async () => {
29+
const stream = await navigator.mediaDevices.getDisplayMedia({
30+
video: true,
31+
audio: true,
32+
})
33+
activeStreams.value.push(stream)
34+
})
35+
}
36+
catch (err) {
37+
console.error('Error selecting source:', err)
38+
}
39+
}
40+
41+
function stopStream(stream: MediaStream) {
42+
stream.getTracks().forEach(track => track.stop())
43+
const index = activeStreams.value.indexOf(stream)
44+
if (index !== -1) {
45+
activeStreams.value.splice(index, 1)
46+
}
47+
}
48+
49+
async function refetchSources() {
50+
try {
51+
isRefetching.value = true
52+
53+
const nextSources = (await getSources())
54+
.sort((a, b) => {
55+
if (a.id.startsWith('screen:') && b.id.startsWith('window:'))
56+
return -1
57+
if (a.id.startsWith('window:') && b.id.startsWith('screen:'))
58+
return 1
59+
return a.name.localeCompare(b.name)
60+
})
61+
62+
sources.value.forEach((oldSource) => {
63+
if (oldSource.appIconURL)
64+
URL.revokeObjectURL(oldSource.appIconURL)
65+
if (oldSource.thumbnailURL)
66+
URL.revokeObjectURL(oldSource.thumbnailURL)
67+
})
68+
69+
sources.value = nextSources.map(source => ({
70+
...source,
71+
appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
72+
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
73+
}))
74+
}
75+
finally {
76+
isRefetching.value = false
77+
}
78+
}
79+
80+
onMounted(async () => {
81+
refetchSources()
82+
})
83+
84+
onBeforeUnmount(() => {
85+
sources.value.forEach((source) => {
86+
if (source.appIconURL)
87+
URL.revokeObjectURL(source.appIconURL)
88+
if (source.thumbnailURL)
89+
URL.revokeObjectURL(source.thumbnailURL)
90+
})
91+
})
92+
</script>
93+
94+
<template>
95+
<div
96+
flex="~ col gap-4 items-start" w-full
97+
text="neutral-500 dark:neutral-400"
98+
>
99+
<div
100+
v-if="activeStreams.length > 0"
101+
bg="primary-300/10"
102+
b="2 solid primary-400/70"
103+
w-full overflow-hidden rounded-2xl p-3
104+
flex="~ col gap-2"
105+
>
106+
<div flex="~ row items-center gap-2">
107+
<div class="i-solar:videocamera-record-line-duotone" />
108+
<div>Capturing</div>
109+
</div>
110+
<div
111+
flex="~ row items-center gap-3"
112+
w-full overflow-x-auto
113+
>
114+
<div
115+
v-for="stream in activeStreams" :key="stream.id"
116+
relative overflow-hidden rounded-lg
117+
>
118+
<div
119+
flex="~ col items-center justify-center gap-1"
120+
absolute right-0 top-0 z-10 h-full w-full cursor-pointer
121+
rounded-lg op-0 backdrop-blur-sm hover:op-100
122+
transition="all duration-200"
123+
text="light"
124+
bg="black/30"
125+
@click="stopStream(stream)"
126+
>
127+
<div class="i-solar:stop-line-duotone" />
128+
<div text-sm>
129+
Stop
130+
</div>
131+
</div>
132+
<video
133+
autoplay
134+
muted
135+
playsinline
136+
:srcObject="stream"
137+
h-140px
138+
w-auto
139+
/>
140+
</div>
141+
</div>
142+
</div>
143+
144+
<div
145+
flex="~ col gap-3"
146+
w-full pb-6
147+
>
148+
<div flex="~ row items-center justify-between" w-full>
149+
<div>{{ sources.length }} source(s)</div>
150+
<Button
151+
:label="isRefetching ? 'Refetching...' : 'Refetch'"
152+
icon="i-solar:refresh-line-duotone"
153+
size="sm"
154+
:disabled="isRefetching"
155+
@click="refetchSources()"
156+
/>
157+
</div>
158+
<div grid="~ cols-1 sm:cols-2 md:cols-3 lg:cols-4 xl:cols-5 gap-3">
159+
<div
160+
v-for="source in sources"
161+
:key="source.id"
162+
flex="~ col justify-between gap-3"
163+
w-full cursor-pointer rounded-2xl p-3
164+
transition="all duration-200"
165+
border="2 solid neutral-200/60 dark:neutral-800/10 hover:primary-400/70"
166+
@click="startCapture(source)"
167+
>
168+
<div flex="~ col items-start w-full">
169+
<div flex="~ row items-center gap-1">
170+
<div class="h-16px w-16px">
171+
<img
172+
v-if="source.appIconURL"
173+
:src="source.appIconURL"
174+
:alt="source.id.startsWith('screen:') ? 'Screen Icon' : 'Window Icon'"
175+
class="h-full w-full shrink-0"
176+
>
177+
<div
178+
v-else-if="source.id.startsWith('screen:')"
179+
h-full w-full
180+
class="i-solar:screencast-2-line-duotone"
181+
/>
182+
<div
183+
v-else
184+
h-full w-full
185+
class="i-solar:window-frame-line-duotone"
186+
/>
187+
</div>
188+
189+
<div text-sm>
190+
{{ source.id.startsWith('screen:') ? 'Screen' : 'Window' }}
191+
</div>
192+
</div>
193+
194+
<div text-ellipsis break-all>
195+
{{ source.name }}
196+
</div>
197+
<div text-sm font-mono text="neutral-400 dark:neutral-600">
198+
{{ source.id }}
199+
</div>
200+
</div>
201+
<div
202+
h-200px w-full overflow-hidden rounded-2xl bg-black
203+
flex="~ items-center justify-center shrink-0"
204+
>
205+
<img
206+
v-if="source.thumbnailURL"
207+
:src="source.thumbnailURL"
208+
alt="Thumbnail"
209+
h-full
210+
w-full object-contain
211+
>
212+
<div
213+
v-else
214+
class="i-solar:forbidden-circle-line-duotone"
215+
h-10 w-10 bg-light
216+
/>
217+
</div>
218+
</div>
219+
</div>
220+
</div>
221+
</div>
222+
</template>
223+
224+
<route lang="yaml">
225+
meta:
226+
layout: settings
227+
</route>

apps/stage-tamagotchi/src/renderer/pages/settings/system/developer.vue

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ const menu = computed(() => [
6565
icon: 'i-solar:transfer-horizontal-bold-duotone',
6666
to: '/devtools/websocket-inspector',
6767
},
68+
{
69+
title: 'Screen Capture',
70+
description: 'Capture screen or window as video and/or audio streams',
71+
icon: 'i-solar:screen-share-bold-duotone',
72+
to: '/devtools/screen-capture',
73+
},
6874
])
6975
7076
const openDevTools = useElectronEventaInvoke(electronOpenMainDevtools)

cspell.config.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ words:
1515
- allseto
1616
- animejs
1717
- APNG
18+
- Armbruster
1819
- astrojs
1920
- Attributify
2021
- attw
@@ -34,6 +35,7 @@ words:
3435
- buildless
3536
- bumpp
3637
- byteorder
38+
- Catap
3739
- catppuccin
3840
- cdylib
3941
- cerebras
@@ -228,6 +230,7 @@ words:
228230
- prismarine
229231
- prng
230232
- pthread
233+
- pulseaudio
231234
- quanlai
232235
- qwen
233236
- Raycaster

0 commit comments

Comments
 (0)