ajax/080609
1
0
Fork 0
mirror of https://github.com/hexahigh/080609.git synced 2025-12-11 19:55:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #93 from hexahigh/terminal-video-webworker

Browse Source 
Terminal video webworker
This commit is contained in:
Simon Bråten 2025-11-28 14:52:47 +01:00 committed by GitHub
commit b788220d4b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 239 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.prettierrc
View File

@ -1,5 +1,5 @@
{
"useTabs": true,
"useTabs": false,
"singleQuote": true,
"trailingComma": "none",
"printWidth": 100,

3
package.json
View File

@ -48,6 +48,7 @@
"remark-rehype": "^11.1.1",
"svelte-meta-tags": "^4.1.0",
"tone": "^15.0.4",
"unified": "^11.0.5"
"unified": "^11.0.5",
"zstddec": "^0.1.0"
}
}

8
pnpm-lock.yaml generated
View File

@ -65,6 +65,9 @@ importers:
unified:
specifier: ^11.0.5
version: 11.0.5
zstddec:
specifier: ^0.1.0
version: 0.1.0
devDependencies:
'@sveltejs/adapter-auto':
specifier: ^3.3.1
@ -1757,6 +1760,9 @@ packages:
zimmerframe@1.1.2:
resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==}
zstddec@0.1.0:
resolution: {integrity: sha512-w2NTI8+3l3eeltKAdK8QpiLo/flRAr2p8AGeakfMZOXBxOg9HIu4LVDxBi81sYgVhFhdJjv1OrB5ssI8uFPoLg==}
zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
@ -3533,4 +3539,6 @@ snapshots:
zimmerframe@1.1.2: {}
zstddec@0.1.0: {}
zwitch@2.0.4: {}

2
src/routes/term/+page.svelte
View File

@ -611,7 +611,7 @@
{#each lineData as line, i (i)}
<span>
{#if line.type === 'input'}
<p class="prompt">{user}@{machine}:$&nbsp;</p>
<p class="prompt">{user}@{machine}:$</p>
<pre class="input-old">{line.command}</pre>
<br />
{:else if line.type === 'output'}

2
src/routes/term/style.css
View File

@ -138,6 +138,8 @@ pre {
.input {
color: var(--color-input);
font-family: var(--font-family);
background: transparent;
@apply p-0
}
.input:focus {

31
src/routes/term/types.ts
View File

@ -48,7 +48,7 @@ export interface TextVideo2 {
fps: number;
width: number;
height: number;
compression: "gzip" | "none" | "";
compression: "gzip" | "zstd" | "none" | "";
}
frames: {
data: Uint8Array;
@ -63,4 +63,31 @@ export type LineDataEntry =
| {
output: string;
type: 'output' | 'outputHtml';
};
};
export type Video2WorkerMessage = {
type: 'init';
video: TextVideo2;
oneBit?: boolean;
} | {
type: 'requestFrame';
index: number;
} | {
type: 'stats';
}
export type Video2WorkerResponse = {
type: 'frame';
index: number;
text: string;
} | {
type: 'end';
} | {
type: 'stats';
stats: {
buffer: {
size: number;
max: number;
}
};
}

84
src/routes/term/video.ts
View File

@ -1,7 +1,7 @@
import type { StdlibType, TextVideo } from "./types";
import * as Tone from "tone";
import axios, { type AxiosResponse } from "axios";
import Pako from "pako";
import type { StdlibType, TextVideo } from './types';
import * as Tone from 'tone';
import axios, { type AxiosResponse } from 'axios';
import Pako from 'pako';
type PlayOptions = {
speed?: number;
@ -11,8 +11,8 @@ type PlayOptions = {
const defaultOptions: PlayOptions = {
speed: 1,
delayToSkip: 100,
}
delayToSkip: 100
};
export async function play(
stdlib: StdlibType,
@ -20,14 +20,13 @@ export async function play(
audioUrl: string,
options: PlayOptions
) {
options = { ...defaultOptions, ...options };
stdlib.print("Loading, please wait...");
stdlib.print('Loading, please wait...');
let player = new Tone.Player();
if (audioUrl !== "") {
if (audioUrl !== '') {
// Load audio
player = new Tone.Player(audioUrl).toDestination();
}
@ -42,34 +41,32 @@ export async function play(
});
if (video.format_version !== 3) {
stdlib.print(
"Unsupported format version, expected 3, got " + video.format_version
);
stdlib.print('Unsupported format version, expected 3, got ' + video.format_version);
return;
}
if (video.encoding !== "" || video.compression !== "") {
if (video.encoding !== '' || video.compression !== '') {
// Decode the encoded frames
stdlib.print("Decoding frames...");
let decodedFrames = "";
stdlib.print('Decoding frames...');
let decodedFrames = '';
switch (video.encoding) {
case "base64":
case 'base64':
decodedFrames = atob(video.encodedFrames);
break;
}
if (!decodedFrames) {
stdlib.print("Error: Decoded frames are undefined.");
stdlib.print('Error: Decoded frames are undefined.');
return;
}
// Decompress the frames
stdlib.print("Decompressing frames...");
stdlib.print('Decompressing frames...');
let decompressedFrames;
switch (video.compression) {
case "gzip":
case 'gzip':
// Convert binary string to character-number array
let charData = decodedFrames.split("").map(function (x) {
let charData = decodedFrames.split('').map(function (x) {
return x.charCodeAt(0);
});
// Turn number array into byte-array
@ -77,28 +74,26 @@ export async function play(
// Pako
let data = Pako.inflate(binData);
// Convert gunzipped byteArray back to ascii string:
decompressedFrames = new TextDecoder("utf-8").decode(data);
decompressedFrames = new TextDecoder('utf-8').decode(data);
break;
}
video.frames = JSON.parse(decompressedFrames);
}
if (!video.width) {
stdlib.print(
"Video width is undefined, guessing from 100th frame. This may be inaccurate."
);
stdlib.print('Video width is undefined, guessing from 100th frame. This may be inaccurate.');
// Split newlines
try {
video.width = video.frames[99].text.split("\n")[0].length;
video.width = video.frames[99].text.split('\n')[0].length;
} catch (e) {
// Fallback to trying the first frame
try {
video.width = video.frames[0].text.split("\n")[0].length;
video.width = video.frames[0].text.split('\n')[0].length;
} catch (e2) {
console.error(
"An error occured while guessing the video width, when trying to get the width of the 100th frame this exception occurred:",
'An error occured while guessing the video width, when trying to get the width of the 100th frame this exception occurred:',
e,
"and when trying to get the width of the first frame this exception occurred:",
'and when trying to get the width of the first frame this exception occurred:',
e2
);
}
@ -108,12 +103,12 @@ export async function play(
// Calculate the scale
scaleFactor = 100 / video.width;
stdlib.print("Scale Factor: " + scaleFactor);
stdlib.print("Fps: " + video.fps);
stdlib.print("Width: " + video.width);
stdlib.print('Scale Factor: ' + scaleFactor);
stdlib.print('Fps: ' + video.fps);
stdlib.print('Width: ' + video.width);
stdlib.print(
"Your video will start in 5 seconds, if the video looks weird then you might need to zoom out."
'Your video will start in 5 seconds, if the video looks weird then you might need to zoom out.'
);
await new Promise((resolve) => setTimeout(resolve, 5000));
@ -142,28 +137,31 @@ export async function play(
// Print one frame every options.speed milliseconds
let i = 0;
const framesLength = video.frames.length; // Store the length of the frames array
let startTime = Date.now()
let skippedInARow: number = 0
let startTime = Date.now();
let skippedInARow: number = 0;
setInterval(() => {
// Check if i is within the bounds of the frames array
if (i < framesLength) {
let sinceStart = Date.now() - startTime
let sinceStart = Date.now() - startTime;
let frame = timeToFrame(video.fps, sinceStart);
let delayMs = Math.abs(frameToTime(video.fps, i).ms - frameToTime(video.fps, frame).ms);
if (delayMs > options.delayToSkip) {
console.log(delayMs, "out of sync. Skipping to", frame)
console.log(delayMs, 'out of sync. Skipping to', frame);
// Skip to the correct frame
i = Math.round(frame)
i = Math.round(frame);
if (skippedInARow >= 5) {
console.log("We seem to be stuck in a loop, setting max delay to:", options.delayToSkip + 10)
options.delayToSkip = options.delayToSkip + 10
skippedInARow = 0
console.log(
'We seem to be stuck in a loop, setting max delay to:',
options.delayToSkip + 10
);
options.delayToSkip = options.delayToSkip + 10;
skippedInARow = 0;
} else {
skippedInARow++
skippedInARow++;
}
return;
} else {
skippedInARow = 0
skippedInARow = 0;
}
// Print the frame
stdlib.setLineData([]);
@ -185,7 +183,7 @@ function frameToTime(fps: number, frame: number) {
return {
ms: ms,
sec: seconds,
sec: seconds
};
}

108
src/routes/term/video2.ts
View File

@ -1,22 +1,21 @@
import type { StdlibType, TextVideo2 } from "./types";
import * as Tone from "tone";
import axios, { type AxiosResponse } from "axios";
import type { StdlibType, TextVideo2, Video2WorkerResponse, Video2WorkerMessage } from './types';
import * as Tone from 'tone';
import { unpack, pack } from 'msgpackr';
import Pako from "pako";
import video2Worker from './video2.worker?worker';
type PlayOptions = {
speed?: number;
// Maximum delay between audio and video before skipping
delayToSkip?: number;
// Fully monochrome
oneBit?: boolean
oneBit?: boolean;
};
const defaultOptions: PlayOptions = {
speed: 1,
delayToSkip: 100,
oneBit: false
}
};
export async function play(
stdlib: StdlibType,
@ -24,14 +23,13 @@ export async function play(
audioUrl: string,
options: PlayOptions
) {
options = { ...defaultOptions, ...options };
stdlib.print("Loading, please wait...");
stdlib.print('Loading, please wait...');
let player = new Tone.Player();
if (audioUrl !== "") {
if (audioUrl !== '') {
// Load audio
player = new Tone.Player(audioUrl).toDestination();
}
@ -41,21 +39,19 @@ export async function play(
let video: TextVideo2;
// Download video
const response = await fetch(videoUrl)
const response = await fetch(videoUrl);
const arrayBuffer = await response.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
video = unpack(uint8Array)
if (video.format_version !== 1) {
stdlib.print(
"Unsupported format version, expected 1, got " + video.format_version
);
video = unpack(uint8Array);
if (video.format_version !== 1 && video.format_version !== 2) {
stdlib.print('Unsupported format version, expected 1 or 2, got ' + video.format_version);
return;
}
// Create pixel-to-char lookup table (LUT)
const chars = options.oneBit ? " #" : " .,-:;=+*#%$@";
const chars = options.oneBit ? ' #' : ' .,-:;=+*#%$@';
const lut = new Array(256);
for (let i = 0; i < 256; i++) {
const index = Math.floor((i / 255) * (chars.length - 1));
@ -65,16 +61,43 @@ export async function play(
// Calculate the scale
scaleFactor = 100 / video.video_info.width;
stdlib.print("Scale Factor: " + scaleFactor);
stdlib.print("Fps: " + video.video_info.fps);
stdlib.print("Width: " + video.video_info.width);
stdlib.print("Height: " + video.video_info.height);
stdlib.print('Scale Factor: ' + scaleFactor);
stdlib.print('Fps: ' + video.video_info.fps);
stdlib.print('Width: ' + video.video_info.width);
stdlib.print('Height: ' + video.video_info.height);
// Create the web worker (make sure the worker file is named "videoWorker.js")
const worker = new video2Worker();
// Receive messages from the worker.
worker.onmessage = (e) => {
const data = e.data as Video2WorkerResponse;
switch (data.type) {
case 'frame':
const frame = data.text;
const lines = frame.split('\n');
stdlib.setLineData([]);
for (let i = 0; i < lines.length; i++) {
stdlib.print(lines[i]);
}
break;
case 'stats':
console.log(`Received stats from worker: ${data.stats}`);
break;
}
};
// Send the video data and oneBit option to the worker for decoding.
worker.postMessage({ type: 'init', video, oneBit: options.oneBit } as Video2WorkerMessage);
stdlib.print(
"Your video will start in 5 seconds, if the video looks weird then you might need to zoom out."
'Your video will start in 5 seconds, if the video looks weird then you might need to zoom out.'
);
await new Promise((resolve) => setTimeout(resolve, 5000));
// Start audio when Tone is loaded.
Tone.loaded().then(() => {
if (options.speed) {
player.playbackRate = options.speed;
@ -100,38 +123,36 @@ export async function play(
// Print one frame every options.speed milliseconds
let i = 0;
const framesLength = video.frames.length; // Store the length of the frames array
let startTime = Date.now()
let skippedInARow: number = 0
let startTime = Date.now();
let skippedInARow: number = 0;
setInterval(() => {
// Check if i is within the bounds of the frames array
if (i < framesLength) {
let sinceStart = Date.now() - startTime
let sinceStart = Date.now() - startTime;
let frame = timeToFrame(video.video_info.fps, sinceStart);
let delayMs = Math.abs(frameToTime(video.video_info.fps, i).ms - frameToTime(video.video_info.fps, frame).ms);
let delayMs = Math.abs(
frameToTime(video.video_info.fps, i).ms - frameToTime(video.video_info.fps, frame).ms
);
if (delayMs > options.delayToSkip) {
console.log(delayMs, "out of sync. Skipping to", frame)
console.log(delayMs, 'out of sync. Skipping to', frame);
// Skip to the correct frame
i = Math.round(frame)
i = Math.round(frame);
if (skippedInARow >= 5) {
console.log("We seem to be stuck in a loop, setting max delay to:", options.delayToSkip + 10)
options.delayToSkip = options.delayToSkip + 10
skippedInARow = 0
console.log(
'We seem to be stuck in a loop, setting max delay to:',
options.delayToSkip + 10
);
options.delayToSkip = options.delayToSkip + 10;
skippedInARow = 0;
} else {
skippedInARow++
skippedInARow++;
}
return;
} else {
skippedInARow = 0
}
let frameData: Uint8Array
if (video.video_info.compression === "gzip") {
frameData = Pako.inflate(video.frames[i].data)
} else {
frameData = video.frames[i].data
skippedInARow = 0;
}
// Print the frame
stdlib.setLineData([]);
stdlib.print(pixelsToChars(frameData, video.video_info.width, video.video_info.height, lut));
worker.postMessage({ type: 'requestFrame', index: i } as Video2WorkerMessage);
i++;
} else {
stdlib.showStuff;
@ -142,14 +163,13 @@ export async function play(
}, delay);
});
}
function frameToTime(fps: number, frame: number) {
let ms = (frame / fps) * 1000;
let seconds = Math.floor(ms / 1000);
return {
ms: ms,
sec: seconds,
sec: seconds
};
}
@ -175,4 +195,4 @@ function pixelsToChars(pixels: Uint8Array, width: number, height: number, lut: s
lines.push(line.join(''));
}
return lines.join('\n');
}
}

91
src/routes/term/video2.worker.ts Normal file
View File

@ -0,0 +1,91 @@
import Pako from 'pako';
import { ZSTDDecoder } from 'zstddec';
import type { Video2WorkerMessage, Video2WorkerResponse, TextVideo2 } from './types';
let frameBuffer: string[] = [];
let video = {} as TextVideo2;
let oneBit = false;
const zstdDecoder = new ZSTDDecoder();
// Build the pixel-to-char lookup table (LUT)
function createLUT(oneBit: boolean): string[] {
const chars = oneBit ? ' #' : ' .,-:;=+*#%$@';
const lut = new Array(256);
for (let i = 0; i < 256; i++) {
const index = Math.floor((i / 255) * (chars.length - 1));
lut[i] = chars[index];
}
return lut;
}
// Convert the raw pixel data to a text frame.
function pixelsToChars(pixels: Uint8Array, width: number, height: number, lut: string[]): string {
const lines: string[] = [];
for (let y = 0; y < height; y++) {
let line = '';
for (let x = 0; x < width; x++) {
line += lut[pixels[y * width + x]];
}
lines.push(line);
}
return lines.join('\n');
}
function addFrameToBuffer(index: number, options: { addEvenIfpresent?: boolean } = {}) {
if (!options.addEvenIfpresent && frameBuffer[index]) {
return;
}
const lut = createLUT(oneBit);
const frame = video.frames[index];
let frameData: Uint8Array;
switch (video.video_info.compression) {
case 'gzip':
frameData = Pako.inflate(frame.data);
break;
case 'zstd':
frameData = zstdDecoder.decode(frame.data, video.video_info.width * video.video_info.height);
break;
default:
frameData = frame.data;
}
const textFrame = pixelsToChars(frameData, video.video_info.width, video.video_info.height, lut);
frameBuffer[index] = textFrame;
}
function add5sOfFramesToBuffer(index: number) {
const totalFrames = video.frames.length;
for (let i = index; i < totalFrames && i < index + 5 * video.video_info.fps; i++) {
addFrameToBuffer(i);
}
}
self.onmessage = async function (e) {
let data = e.data as Video2WorkerMessage;
switch (data.type) {
case 'init':
video = data.video;
oneBit = data.oneBit || false;
if (data.video.video_info.compression === 'zstd') {
await zstdDecoder.init();
}
add5sOfFramesToBuffer(0);
break;
case 'requestFrame':
if (!frameBuffer[data.index]) {
addFrameToBuffer(data.index);
}
let frame = frameBuffer[data.index];
postMessage({ type: 'frame', index: data.index, text: frame });
add5sOfFramesToBuffer(data.index + 1);
break;
case 'stats':
postMessage({
type: 'stats',
stats: { buffer: { size: frameBuffer.length, max: video.frames.length } }
} as Video2WorkerResponse);
}
};
export {};