From 7cbbb709ebfe85ed34301264e0628cf3b15a2338 Mon Sep 17 00:00:00 2001 From: Sebastian Jeong Date: Sun, 16 Aug 2026 16:19:19 +0900 Subject: [PATCH] feat: consolidate VM automation in Python daemon --- .pi/extensions/dos-vm.md | 20 --- .pi/extensions/dos-vm.ts | 222 ---------------------------- .qemu/monitor.ps1 | 36 ----- .qemu/reset.ps1 | 68 --------- .qemu/run.ps1 | 37 ----- .qemu/screenshot.ps1 | 26 ---- .qemu/tcp-agent-relay.mjs | 71 --------- pyproject.toml | 10 ++ src/ferrolang_vm/__init__.py | 1 + src/ferrolang_vm/cli.py | 99 +++++++++++++ src/ferrolang_vm/daemon.py | 270 +++++++++++++++++++++++++++++++++++ tools/README.md | 50 +++++-- tools/dos_stage.py | 122 ---------------- tools/qemu_ocr.py | 10 +- tools/tcpagent/README.md | 25 ++-- uv.lock | 2 +- 16 files changed, 434 insertions(+), 635 deletions(-) delete mode 100644 .pi/extensions/dos-vm.md delete mode 100644 .pi/extensions/dos-vm.ts delete mode 100644 .qemu/monitor.ps1 delete mode 100644 .qemu/reset.ps1 delete mode 100644 .qemu/run.ps1 delete mode 100644 .qemu/screenshot.ps1 delete mode 100644 .qemu/tcp-agent-relay.mjs create mode 100644 src/ferrolang_vm/__init__.py create mode 100644 src/ferrolang_vm/cli.py create mode 100644 src/ferrolang_vm/daemon.py delete mode 100644 tools/dos_stage.py diff --git a/.pi/extensions/dos-vm.md b/.pi/extensions/dos-vm.md deleted file mode 100644 index 5893081..0000000 --- a/.pi/extensions/dos-vm.md +++ /dev/null @@ -1,20 +0,0 @@ -# DOS VM tools - -This project-local Pi extension exposes `dos_read`, `dos_write`, `dos_edit`, -`dos_list`, and `dos_exec`. It communicates with `DOSAGENT.EXE` over COM1 -through the serial relay controller port at `127.0.0.1:5555`. - -The agent is not started automatically. From the FreeDOS VGA console, run -`DOSAGENT` (installed in `C:\\FREEDOS\\BIN`, which is on `PATH`) when Pi tool -access is needed. While it runs, press Ctrl+C in the VGA console to stop it and -return to the DOS prompt; Pi tools then cannot connect until it is started -again. QEMU monitor control remains available at `127.0.0.1:4444`. - -Connect PuTTY in Raw mode to `127.0.0.1:5557` for a read-only decoded mirror of -Pi/DOS serial traffic. Port 5556 is reserved for the internal QEMU-to-relay -connection. - -`D:` is QEMU's FAT-directory view of `.qemu\\share`. Treat it as an exchange -volume, not a live-sync mount: host-side changes can be stale or partially -visible to a running VM. Restart QEMU before compiling host-edited files, or -use `dos_write` to place the source on `C:` before compiling. diff --git a/.pi/extensions/dos-vm.ts b/.pi/extensions/dos-vm.ts deleted file mode 100644 index 3721710..0000000 --- a/.pi/extensions/dos-vm.ts +++ /dev/null @@ -1,222 +0,0 @@ -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { Type } from "typebox"; -import net from "node:net"; - -const HOST = "127.0.0.1"; -const PORT = 5555; -const MONITOR_PORT = 4444; -const CHUNK_SIZE = 4096; -const MAX_FILE_SIZE = 4 * 1024 * 1024; - -function hexEncode(value: string | Buffer): string { - return Buffer.from(value).toString("hex").toUpperCase(); -} - -function hexDecode(value: string): Buffer { - return Buffer.from(value, "hex"); -} - -function normalizePath(path: string): string { - const result = path.startsWith("@") ? path.slice(1) : path; - if (!/^[A-Za-z]:[\\/]/.test(result)) { - throw new Error(`DOS path must be absolute (for example C:\\SRC\\FILE.C): ${result}`); - } - return result.replaceAll("/", "\\"); -} - -async function request(command: string, signal?: AbortSignal, timeoutMs = 30_000): Promise { - return new Promise((resolve, reject) => { - const socket = net.createConnection({ host: HOST, port: PORT }); - let buffered = ""; - let settled = false; - - const finish = (error?: Error, value?: string) => { - if (settled) return; - settled = true; - clearTimeout(timer); - signal?.removeEventListener("abort", abort); - socket.destroy(); - if (error) reject(error); else resolve(value ?? ""); - }; - const abort = () => finish(new Error("DOS tool call aborted")); - const timer = setTimeout(() => finish(new Error(`DOS agent timed out after ${timeoutMs}ms`)), timeoutMs); - - signal?.addEventListener("abort", abort, { once: true }); - socket.setEncoding("ascii"); - socket.on("connect", () => socket.write(`${command}\r\n`, "ascii")); - socket.on("data", (chunk) => { - buffered += chunk; - const lines = buffered.split(/\r?\n/); - buffered = lines.pop() ?? ""; - for (const line of lines) { - if (line.startsWith("OK ")) return finish(undefined, line.slice(3)); - if (line === "OK") return finish(undefined, ""); - if (line.startsWith("ERR ")) return finish(new Error(hexDecode(line.slice(4)).toString("utf8"))); - } - }); - socket.on("error", (error) => finish(new Error(`Cannot connect to DOS agent at ${HOST}:${PORT}: ${error.message}`))); - socket.on("close", () => { - if (!settled) finish(new Error("DOS serial connection closed before a response")); - }); - }); -} - -async function interruptForegroundCommand(): Promise { - await new Promise((resolve, reject) => { - const socket = net.createConnection({ host: HOST, port: MONITOR_PORT }); - const timer = setTimeout(() => { socket.destroy(); reject(new Error("QEMU monitor did not accept Ctrl+C")); }, 5_000); - socket.on("connect", () => socket.write("sendkey ctrl-c\r\n", "ascii")); - socket.on("data", () => { clearTimeout(timer); socket.destroy(); resolve(); }); - socket.on("error", (error) => { clearTimeout(timer); reject(new Error(`Cannot contact QEMU monitor: ${error.message}`)); }); - }); -} - -async function waitForAgentRecovery(timeoutMs = 10_000): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - try { - if (await request("PING", undefined, 1_000) === "504F4E47") return true; - } catch { /* Command is still unwinding. */ } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - return false; -} - -async function readFile(path: string, signal?: AbortSignal): Promise { - const dosPath = normalizePath(path); - const chunks: Buffer[] = []; - let size = 0; - let offset = 0; - for (;;) { - const response = await request(`READ ${hexEncode(dosPath)} ${offset}`, signal); - const match = /^([01])(?:\s(.*))?$/.exec(response); - if (!match) throw new Error(`Malformed READ response: ${response}`); - const chunk = hexDecode(match[2] ?? ""); - chunks.push(chunk); - size += chunk.length; - if (size > MAX_FILE_SIZE) throw new Error(`DOS file exceeds ${MAX_FILE_SIZE} byte tool limit`); - offset += chunk.length; - if (match[1] === "1") return Buffer.concat(chunks, size); - if (chunk.length === 0) throw new Error("DOS agent returned an empty non-final chunk"); - } -} - -async function writeFile(path: string, content: Buffer, signal?: AbortSignal): Promise { - const dosPath = normalizePath(path); - if (content.length > MAX_FILE_SIZE) throw new Error(`Content exceeds ${MAX_FILE_SIZE} byte tool limit`); - if (content.length === 0) { - await request(`WRITE ${hexEncode(dosPath)} T `, signal); - return; - } - for (let offset = 0; offset < content.length; offset += CHUNK_SIZE) { - const chunk = content.subarray(offset, offset + CHUNK_SIZE); - await request(`WRITE ${hexEncode(dosPath)} ${offset === 0 ? "T" : "A"} ${hexEncode(chunk)}`, signal); - } -} - -function textResult(text: string) { - return { content: [{ type: "text" as const, text }], details: {} }; -} - -export default function (pi: ExtensionAPI) { - pi.registerTool({ - name: "dos_read", - label: "DOS Read", - description: "Read a text file from the running FreeDOS VM over COM1.", - promptSnippet: "Read files inside the running FreeDOS VM", - parameters: Type.Object({ path: Type.String({ description: "Absolute DOS path such as C:\\SRC\\MAIN.C" }) }), - async execute(_id, params, signal) { - const content = await readFile(params.path, signal); - return textResult(content.toString("utf8")); - }, - }); - - pi.registerTool({ - name: "dos_write", - label: "DOS Write", - description: "Create or replace a text file in the running FreeDOS VM over COM1.", - promptSnippet: "Write files inside the running FreeDOS VM", - parameters: Type.Object({ - path: Type.String({ description: "Absolute DOS path" }), - content: Type.String({ description: "Complete UTF-8 text content" }), - }), - async execute(_id, params, signal) { - const content = Buffer.from(params.content, "utf8"); - await writeFile(params.path, content, signal); - return textResult(`Wrote ${content.length} bytes to ${normalizePath(params.path)}`); - }, - }); - - pi.registerTool({ - name: "dos_edit", - label: "DOS Edit", - description: "Replace one uniquely matching text fragment in a file in the running FreeDOS VM.", - promptSnippet: "Make exact text replacements inside FreeDOS files", - parameters: Type.Object({ - path: Type.String({ description: "Absolute DOS path" }), - old_text: Type.String({ description: "Text that must occur exactly once" }), - new_text: Type.String({ description: "Replacement text" }), - }), - async execute(_id, params, signal) { - const original = (await readFile(params.path, signal)).toString("utf8"); - const occurrences = original.split(params.old_text).length - 1; - if (occurrences !== 1) throw new Error(`Expected old_text exactly once, found ${occurrences} occurrences`); - const updated = original.replace(params.old_text, params.new_text); - await writeFile(params.path, Buffer.from(updated, "utf8"), signal); - return textResult(`Edited ${normalizePath(params.path)}`); - }, - }); - - pi.registerTool({ - name: "dos_list", - label: "DOS List", - description: "List a directory in the running FreeDOS VM over COM1.", - promptSnippet: "List directories inside the running FreeDOS VM", - parameters: Type.Object({ path: Type.String({ description: "Absolute DOS directory path" }) }), - async execute(_id, params, signal) { - const response = await request(`LIST ${hexEncode(normalizePath(params.path))}`, signal); - return textResult(hexDecode(response).toString("utf8")); - }, - }); - - pi.registerTool({ - name: "dos_exec", - label: "DOS Exec", - description: "Execute a command in the running FreeDOS VM and return its output.", - promptSnippet: "Run commands inside the running FreeDOS VM", - parameters: Type.Object({ - command: Type.String({ description: "FreeDOS command line" }), - timeout_seconds: Type.Optional(Type.Integer({ minimum: 1, maximum: 120, description: "Abort after this many seconds (default: 30)" })), - }), - async execute(_id, params, signal) { - if (params.command.length > 600) throw new Error("DOS command exceeds 600 characters"); - const timeoutMs = (params.timeout_seconds ?? 30) * 1_000; - let response: string; - try { - response = await request(`EXEC ${hexEncode(params.command)}`, signal, timeoutMs); - } catch (error) { - if (signal?.aborted) throw error; - await interruptForegroundCommand(); - const recovered = await waitForAgentRecovery(); - throw new Error(`DOS command timed out after ${timeoutMs / 1_000}s and was interrupted with Ctrl+C${recovered ? "; agent recovered" : "; agent did not recover"}`); - } - const match = /^(-?\d+)(?:\s(.*))?$/.exec(response); - if (!match) throw new Error(`Malformed EXEC response: ${response}`); - const output = hexDecode(match[2] ?? "").toString("utf8"); - return textResult(`${output}${output.endsWith("\n") || !output ? "" : "\n"}[exit ${match[1]}]`); - }, - }); - - pi.registerTool({ - name: "dos_abort", - label: "DOS Abort", - description: "Send Ctrl+C to interrupt the foreground DOS command without rebooting the VM.", - promptSnippet: "Interrupt a hung DOS command without rebooting the VM", - parameters: Type.Object({}), - async execute() { - await interruptForegroundCommand(); - const recovered = await waitForAgentRecovery(); - return textResult(recovered ? "Sent Ctrl+C; DOS agent recovered." : "Sent Ctrl+C; DOS agent has not responded yet."); - }, - }); -} diff --git a/.qemu/monitor.ps1 b/.qemu/monitor.ps1 deleted file mode 100644 index 6a0322f..0000000 --- a/.qemu/monitor.ps1 +++ /dev/null @@ -1,36 +0,0 @@ -param( - [Parameter(Mandatory = $true, Position = 0)] - [string]$Command -) - -$ErrorActionPreference = 'Stop' -$client = [System.Net.Sockets.TcpClient]::new() - -try { - $client.Connect('127.0.0.1', 4444) - $stream = $client.GetStream() - $stream.ReadTimeout = 1000 - $writer = [System.IO.StreamWriter]::new($stream, [System.Text.Encoding]::ASCII, 1024, $true) - $writer.NewLine = "`n" - $writer.AutoFlush = $true - - Start-Sleep -Milliseconds 100 - while ($stream.DataAvailable) { - $buffer = New-Object byte[] 4096 - [void]$stream.Read($buffer, 0, $buffer.Length) - } - - $writer.WriteLine($Command) - Start-Sleep -Milliseconds 200 - - $result = New-Object System.Text.StringBuilder - while ($stream.DataAvailable) { - $buffer = New-Object byte[] 4096 - $count = $stream.Read($buffer, 0, $buffer.Length) - [void]$result.Append([System.Text.Encoding]::ASCII.GetString($buffer, 0, $count)) - } - $result.ToString().Trim() -} -finally { - $client.Dispose() -} diff --git a/.qemu/reset.ps1 b/.qemu/reset.ps1 deleted file mode 100644 index f2465ac..0000000 --- a/.qemu/reset.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -param( - [int] $ReadyTimeoutSeconds = 45 -) - -$ErrorActionPreference = 'Stop' -$root = Split-Path -Parent $MyInvocation.MyCommand.Path - -function Stop-Listener([int] $Port) { - $listeners = Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction SilentlyContinue - foreach ($listener in $listeners) { - Stop-Process -Id $listener.OwningProcess -Force -ErrorAction SilentlyContinue - } -} - -# Quit through QEMU's monitor so qcow2/FAT writes are flushed. Forced VM -# termination is deliberately forbidden because it previously lost files. -$monitor = Get-NetTCPConnection -State Listen -LocalPort 4444 -ErrorAction SilentlyContinue -if ($monitor) { - & (Join-Path $root 'monitor.ps1') 'quit' | Out-Null - $deadline = (Get-Date).AddSeconds(10) - do { - Start-Sleep -Milliseconds 250 - $monitor = Get-NetTCPConnection -State Listen -LocalPort 4444 -ErrorAction SilentlyContinue - } while ($monitor -and (Get-Date) -lt $deadline) - if ($monitor) { throw 'QEMU did not quit cleanly; refusing a forced reset.' } -} -Stop-Listener 5555 -Start-Sleep -Milliseconds 500 - -$runner = Join-Path $root 'run.ps1' -Start-Process powershell -ArgumentList @('-NoProfile','-ExecutionPolicy','Bypass','-File',$runner) -WorkingDirectory $root -$deadline = (Get-Date).AddSeconds(15) -do { - Start-Sleep -Milliseconds 250 - $monitor = Get-NetTCPConnection -State Listen -LocalPort 4444 -ErrorAction SilentlyContinue -} while (-not $monitor -and (Get-Date) -lt $deadline) -if (-not $monitor) { throw 'QEMU monitor did not start.' } - -# Select the default boot entry. TCPAGENT is started by FDAUTO.BAT after the -# packet driver, so readiness is the agent connection/PONG rather than sleep. -Start-Sleep -Seconds 2 -& (Join-Path $root 'monitor.ps1') 'sendkey ret' | Out-Null -$deadline = (Get-Date).AddSeconds($ReadyTimeoutSeconds) -$ready = $false -do { - Start-Sleep -Milliseconds 500 - $connection = Get-NetTCPConnection -State Established -LocalPort 5558 -ErrorAction SilentlyContinue - if ($connection) { - try { - $client = [Net.Sockets.TcpClient]::new('127.0.0.1',5555) - $stream = $client.GetStream() - $stream.ReadTimeout = 1500 - $bytes = [Text.Encoding]::ASCII.GetBytes("PING`n") - $stream.Write($bytes,0,$bytes.Length) - $buffer = New-Object byte[] 128 - $reply = '' - while ($reply -notmatch "`n") { - $count = $stream.Read($buffer,0,$buffer.Length) - if ($count -le 0) { break } - $reply += [Text.Encoding]::ASCII.GetString($buffer,0,$count) - } - $client.Dispose() - $ready = $reply -match '^OK 504F4E47' - } catch { $ready = $false } - } -} while (-not $ready -and (Get-Date) -lt $deadline) -if (-not $ready) { throw 'TCPAGENT did not become ready.' } -Write-Host 'QEMU reset complete; TCPAGENT answered PONG.' diff --git a/.qemu/run.ps1 b/.qemu/run.ps1 deleted file mode 100644 index eefe499..0000000 --- a/.qemu/run.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -param() - -$ErrorActionPreference = 'Stop' -$qemu = Get-Command qemu-system-i386.exe -ErrorAction Stop -$node = Get-Command node.exe -ErrorAction Stop -$disk = Join-Path $PSScriptRoot 'freedos.qcow2' -$relay = Join-Path $PSScriptRoot 'tcp-agent-relay.mjs' - -if (-not (Test-Path -LiteralPath $disk)) { - throw "Missing $disk. Run .qemu\\setup.ps1, then .qemu\\install.ps1." -} - -$relayListening = Get-NetTCPConnection -State Listen -LocalPort 5558 -ErrorAction SilentlyContinue -if (-not $relayListening) { - Start-Process -FilePath $node.Source -ArgumentList @($relay) -WorkingDirectory $PSScriptRoot -WindowStyle Hidden - $deadline = [DateTime]::UtcNow.AddSeconds(5) - do { - Start-Sleep -Milliseconds 100 - $relayListening = Get-NetTCPConnection -State Listen -LocalPort 5558 -ErrorAction SilentlyContinue - } while (-not $relayListening -and [DateTime]::UtcNow -lt $deadline) - if (-not $relayListening) { throw 'TCP agent relay did not start.' } -} - -$monitorListening = Get-NetTCPConnection -State Listen -LocalPort 4444 -ErrorAction SilentlyContinue -if ($monitorListening) { - throw 'QEMU monitor port 4444 is already in use. Stop the existing VM before starting another one.' -} - -& $qemu.Source ` - -machine pc,accel=whpx,kernel-irqchip=off,usb=on ` - -smp 1 ` - -m 64 ` - -drive "file=$disk,format=qcow2,if=ide,index=0,media=disk" ` - -nic user,model=ne2k_isa ` - -monitor "tcp:127.0.0.1:4444,server=on,wait=off" ` - -boot order=c ` - -display default diff --git a/.qemu/screenshot.ps1 b/.qemu/screenshot.ps1 deleted file mode 100644 index bd5c93d..0000000 --- a/.qemu/screenshot.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -param( - [string]$Name = 'qemu-screen', - [switch]$KeepPpm -) - -$ErrorActionPreference = 'Stop' -$ffmpeg = Get-Command ffmpeg.exe -ErrorAction Stop - -$target = if ([IO.Path]::IsPathRooted($Name)) { $Name } else { Join-Path $PSScriptRoot $Name } -$extension = [IO.Path]::GetExtension($target) -if ($extension -in @('.png', '.ppm')) { - $target = [IO.Path]::Combine([IO.Path]::GetDirectoryName($target), [IO.Path]::GetFileNameWithoutExtension($target)) -} -$directory = [IO.Path]::GetDirectoryName($target) -if ($directory) { New-Item -ItemType Directory -Force -Path $directory | Out-Null } - -$ppm = "$target.ppm" -$png = "$target.png" -$qemuPath = $ppm.Replace('\', '/') - -& (Join-Path $PSScriptRoot 'monitor.ps1') "screendump $qemuPath" | Out-Null -if (-not (Test-Path -LiteralPath $ppm)) { throw 'QEMU did not create the requested screenshot.' } -& $ffmpeg.Source -y -loglevel error -i $ppm $png -if ($LASTEXITCODE -ne 0) { throw "ffmpeg conversion failed with exit code $LASTEXITCODE." } -if (-not $KeepPpm) { Remove-Item -LiteralPath $ppm -Force } -Get-Item -LiteralPath $png diff --git a/.qemu/tcp-agent-relay.mjs b/.qemu/tcp-agent-relay.mjs deleted file mode 100644 index 2768bc4..0000000 --- a/.qemu/tcp-agent-relay.mjs +++ /dev/null @@ -1,71 +0,0 @@ -import net from "node:net"; - -const HOST = "127.0.0.1"; -const TOOL_PORT = 5555; -const OBSERVER_PORT = 5557; -const TCP_AGENT_PORT = 5558; - -let tcpAgent = null; -let controller = null; -const observers = new Set(); - -function safeWrite(socket, data) { - if (socket && !socket.destroyed && socket.writable) socket.write(data); -} - -function trace(source, data) { - const text = data.toString("ascii"); - const shortened = text.length > 400 ? `${text.slice(0, 400)}…` : text; - for (const observer of observers) safeWrite(observer, `${source} ${shortened}\r\n`); -} - -const tcpAgentServer = net.createServer((socket) => { - if (tcpAgent && !tcpAgent.destroyed) { - socket.end(); - return; - } - tcpAgent = socket; - socket.setNoDelay(true); - socket.on("data", (data) => { - safeWrite(controller, data); - trace("DOS->Pi", data); - }); - socket.on("close", () => { - if (tcpAgent === socket) tcpAgent = null; - }); - socket.on("error", () => {}); -}); - -const toolServer = net.createServer((socket) => { - if (controller && !controller.destroyed) { - socket.end("ERR 544350204147454E5420434F4E54524F4C4C45522042555359\r\n"); - return; - } - if (!tcpAgent || tcpAgent.destroyed) { - socket.end("ERR 544350204147454E54204E4F5420434F4E4E4543544544\r\n"); - return; - } - controller = socket; - socket.setNoDelay(true); - socket.on("data", (data) => { - safeWrite(tcpAgent, data); - trace("Pi->DOS", data); - }); - socket.on("close", () => { - if (controller === socket) controller = null; - }); - socket.on("error", () => {}); -}); - -const observerServer = net.createServer((socket) => { - observers.add(socket); - socket.setNoDelay(true); - socket.write("DOS TCP agent observer (read-only)\r\n"); - socket.on("data", () => {}); - socket.on("close", () => observers.delete(socket)); - socket.on("error", () => observers.delete(socket)); -}); - -tcpAgentServer.listen(TCP_AGENT_PORT, HOST); -toolServer.listen(TOOL_PORT, HOST); -observerServer.listen(OBSERVER_PORT, HOST); diff --git a/pyproject.toml b/pyproject.toml index 6abdccd..2660113 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,13 @@ dependencies = [ "onnxruntime>=1.28.0", "rapidocr>=3.9.2", ] + +[project.scripts] +ferro-vm = "ferrolang_vm.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/ferrolang_vm"] diff --git a/src/ferrolang_vm/__init__.py b/src/ferrolang_vm/__init__.py new file mode 100644 index 0000000..039a606 --- /dev/null +++ b/src/ferrolang_vm/__init__.py @@ -0,0 +1 @@ +"""Windows-only QEMU and FreeDOS TCP-agent automation.""" diff --git a/src/ferrolang_vm/cli.py b/src/ferrolang_vm/cli.py new file mode 100644 index 0000000..28aa392 --- /dev/null +++ b/src/ferrolang_vm/cli.py @@ -0,0 +1,99 @@ +"""Command line client for the Windows-only ferro-vm daemon.""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from multiprocessing.connection import Client +from pathlib import Path + +from .daemon import PIPE, ROOT + + +def rpc(payload: dict[str, object], start_daemon: bool = False) -> object: + try: + conn = Client(PIPE, family="AF_PIPE") + except (FileNotFoundError, OSError): + if not start_daemon: + raise RuntimeError("ferro-vm daemon is not running; run `uv run ferro-vm start`") + flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) | getattr(subprocess, "DETACHED_PROCESS", 0) + subprocess.Popen([sys.executable, "-m", "ferrolang_vm.daemon"], cwd=ROOT, creationflags=flags, close_fds=True) + deadline = time.monotonic() + 5 + while True: + try: + conn = Client(PIPE, family="AF_PIPE") + break + except (FileNotFoundError, OSError): + if time.monotonic() >= deadline: + raise RuntimeError("ferro-vm daemon did not create its control pipe") + time.sleep(.1) + with conn: + conn.send(payload) + response = conn.recv() + if not response["ok"]: + raise RuntimeError(response["error"]) + return response["result"] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Windows-only QEMU/FreeDOS automation") + commands = parser.add_subparsers(dest="op", required=True) + for name in ("start", "stop", "status", "ping", "screenshot", "ocr"): + commands.add_parser(name) + reset = commands.add_parser("reset") + reset.add_argument("--timeout", type=int, default=45) + execute = commands.add_parser("exec") + execute.add_argument("command") + put = commands.add_parser("put") + put.add_argument("source", type=Path) + put.add_argument("destination") + get = commands.add_parser("get") + get.add_argument("source") + get.add_argument("destination", type=Path) + args = parser.parse_args() + + try: + if args.op == "ocr": + result = rpc({"op": "screenshot"}) + import logging + logging.disable(logging.INFO) + from rapidocr import RapidOCR + recognized = RapidOCR()(result["path"]) + print("\n".join(recognized.txts or ())) + return 0 + if args.op == "reset": + rpc({"op": "stop"}, start_daemon=True) + time.sleep(.5) + rpc({"op": "start"}, start_daemon=True) + # FreeDOS displays its default boot menu before FDAUTO.BAT starts + # TCPAGENT. This is input, not a readiness delay. + time.sleep(2) + rpc({"op": "monitor", "command": "sendkey ret"}) + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + try: + if str(rpc({"op": "ping"})["response"]).startswith("OK 504F4E47"): + print(json.dumps({"reset": True, "agent": "PONG"})) + return 0 + except RuntimeError: + time.sleep(.5) + raise RuntimeError("TCPAGENT did not become ready") + payload: dict[str, object] = {"op": args.op} + if args.op == "exec": payload["command"] = args.command + if args.op == "put": + payload["source"] = str(args.source.resolve()) + payload["destination"] = args.destination + if args.op == "get": + payload["source"] = args.source + payload["destination"] = str(args.destination.resolve()) + print(json.dumps(rpc(payload, start_daemon=args.op == "start"), ensure_ascii=False, indent=2)) + return 0 + except RuntimeError as exc: + print(f"ferro-vm: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ferrolang_vm/daemon.py b/src/ferrolang_vm/daemon.py new file mode 100644 index 0000000..84ab4bd --- /dev/null +++ b/src/ferrolang_vm/daemon.py @@ -0,0 +1,270 @@ +"""Long-lived Windows host for the FreeDOS TCP agent. + +The only automation TCP listener is 127.0.0.1:5558, used exclusively by +TCPAGENT.EXE. Local commands use a Windows named pipe. +""" +from __future__ import annotations + +import json +import logging +import os +import select +import shutil +import socket +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from multiprocessing.connection import Listener +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +QEMU = ROOT / ".qemu" +PIPE = r"\\.\pipe\ferrolang-vm" +AGENT_ADDRESS = ("127.0.0.1", 5558) +MONITOR_ADDRESS = ("127.0.0.1", 4444) +LOG_PATH = QEMU / "ferro-vm.log" + + +def configure_logging() -> None: + QEMU.mkdir(exist_ok=True) + handler = logging.FileHandler(LOG_PATH, encoding="utf-8") + handler.setFormatter(logging.Formatter("%(asctime)s.%(msecs)03dZ %(levelname)-7s %(message)s", "%Y-%m-%dT%H:%M:%S")) + logging.basicConfig(level=logging.INFO, handlers=[handler]) + logging.Formatter.converter = time.gmtime + + +def log_event(level: int, event: str, **fields: object) -> None: + suffix = " ".join(f"{key}={json.dumps(value, ensure_ascii=False)}" for key, value in fields.items()) + logging.log(level, "%s%s", event, f" {suffix}" if suffix else "") + + +class Host: + def __init__(self) -> None: + self.agent: socket.socket | None = None + self.agent_lock = threading.Lock() + self.agent_ready = threading.Event() + self.qemu: subprocess.Popen[bytes] | None = None + + def accept_agents(self) -> None: + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(AGENT_ADDRESS) + server.listen(1) + log_event(logging.INFO, "agent listener ready", address="127.0.0.1:5558") + while True: + sock, peer = server.accept() + sock.settimeout(30) + with self.agent_lock: + if self.agent is not None: + sock.close() + log_event(logging.WARNING, "agent rejected", peer=str(peer), reason="already connected") + continue + self.agent = sock + self.agent_ready.set() + try: + banner = self._read_line(sock).decode("ascii", "replace") + log_event(logging.INFO, "agent connected", peer=f"{peer[0]}:{peer[1]}", banner=banner) + # Request() owns protocol reads. Between requests, peek only + # for EOF so TCPAGENT can reconnect without being rejected. + while self.agent is sock: + if self.agent_lock.acquire(blocking=False): + try: + readable, _, _ = select.select([sock], [], [], .25) + if readable and not sock.recv(1, socket.MSG_PEEK): + break + finally: + self.agent_lock.release() + else: + time.sleep(.05) + finally: + with self.agent_lock: + if self.agent is sock: + self.agent = None + self.agent_ready.clear() + sock.close() + log_event(logging.INFO, "agent disconnected", peer=f"{peer[0]}:{peer[1]}") + + @staticmethod + def _read_line(sock: socket.socket) -> bytes: + out = bytearray() + while not out.endswith(b"\n"): + part = sock.recv(1) + if not part: + raise ConnectionError("TCP agent closed connection") + out.extend(part) + return bytes(out).rstrip(b"\r\n") + + def request(self, command: str, payload: bytes = b"") -> str: + with self.agent_lock: + if self.agent is None: + raise RuntimeError("TCPAGENT is not connected") + sock = self.agent + started = time.monotonic() + try: + sock.sendall(command.encode("ascii") + b"\n" + payload) + response = self._read_line(sock).decode("ascii", "replace") + except OSError as exc: + if self.agent is sock: + self.agent = None + self.agent_ready.clear() + raise RuntimeError(f"TCPAGENT request failed: {exc}") from exc + log_event(logging.INFO, "agent request", command=command.split(" ", 1)[0], response=response[:200], elapsed_ms=round((time.monotonic()-started)*1000)) + if response.startswith("ERR "): + raise RuntimeError(bytes.fromhex(response[4:]).decode("utf-8", "replace")) + return response + + def ping(self) -> dict[str, object]: + return {"response": self.request("PING")} + + def exec(self, command: str) -> dict[str, object]: + log_event(logging.INFO, "exec start", command=command) + response = self.request("EXEC " + command.encode("ascii", "replace").hex().upper()) + fields = response.split(" ", 2) + code = int(fields[1]) if len(fields) > 1 else -1 + output = bytes.fromhex(fields[2]).decode("cp437", "replace") if len(fields) > 2 else "" + for line in output.splitlines(): + log_event(logging.INFO, "dos output", line=line) + log_event(logging.INFO, "exec finish", exit=code) + return {"exit": code, "output": output} + + def put(self, source: str, destination: str) -> dict[str, object]: + data = Path(source).read_bytes() + encoded = destination.encode("ascii").hex().upper() + self.request(f"PUT {encoded} {len(data)}", data) + stat = self.request(f"HASH {encoded}") + log_event(logging.INFO, "put", path=destination, bytes=len(data), stat=stat) + return {"path": destination, "bytes": len(data), "stat": stat} + + def get(self, source: str, destination: str) -> dict[str, object]: + encoded = source.encode("ascii").hex().upper() + with self.agent_lock: + if self.agent is None: + raise RuntimeError("TCPAGENT is not connected") + sock = self.agent + sock.sendall(f"GET {encoded}\n".encode("ascii")) + header = self._read_line(sock).decode("ascii", "strict").split() + if len(header) != 2 or header[0] != "DATA": + raise RuntimeError("GET failed: " + " ".join(header)) + remaining = int(header[1]) + chunks: list[bytes] = [] + while remaining: + chunk = sock.recv(min(65536, remaining)) + if not chunk: + raise RuntimeError("TCPAGENT closed during GET") + chunks.append(chunk) + remaining -= len(chunk) + target = Path(destination) + target.parent.mkdir(parents=True, exist_ok=True) + data = b"".join(chunks) + target.write_bytes(data) + log_event(logging.INFO, "get", path=source, bytes=len(data), destination=str(target)) + return {"path": source, "destination": str(target), "bytes": len(data)} + + @staticmethod + def monitor(command: str) -> str: + with socket.create_connection(MONITOR_ADDRESS, timeout=3) as sock: + sock.settimeout(1) + time.sleep(.1) + try: + sock.recv(4096) + except TimeoutError: + pass + sock.sendall(command.encode("ascii") + b"\n") + time.sleep(.2) + chunks: list[bytes] = [] + while True: + try: + chunk = sock.recv(4096) + except TimeoutError: + break + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks).decode("ascii", "replace").strip() + + def start(self) -> dict[str, object]: + if self.qemu is not None and self.qemu.poll() is None: + return {"started": False, "reason": "already running"} + try: + self.monitor("info status") + return {"started": False, "reason": "already running (external)"} + except OSError: + pass + executable = shutil.which("qemu-system-i386.exe") + disk = QEMU / "freedos.qcow2" + if not executable: + raise RuntimeError("qemu-system-i386.exe is not on PATH") + if not disk.exists(): + raise RuntimeError(f"missing {disk}; run .qemu/setup.ps1 and .qemu/install.ps1") + self.qemu = subprocess.Popen([executable, "-machine", "pc,accel=whpx,kernel-irqchip=off,usb=on", "-smp", "1", "-m", "64", "-drive", f"file={disk},format=qcow2,if=ide,index=0,media=disk", "-nic", "user,model=ne2k_isa", "-monitor", "tcp:127.0.0.1:4444,server=on,wait=off", "-boot", "order=c", "-display", "default"], cwd=QEMU) + log_event(logging.INFO, "qemu started", pid=self.qemu.pid) + return {"started": True, "pid": self.qemu.pid} + + def stop(self) -> dict[str, object]: + try: + self.monitor("quit") + log_event(logging.INFO, "qemu stop requested") + except OSError: + pass + return {"stopped": True} + + def screenshot(self) -> dict[str, object]: + ppm, png = QEMU / "qemu-screen.ppm", QEMU / "qemu-screen.png" + self.monitor("screendump " + str(ppm).replace("\\", "/")) + ffmpeg = shutil.which("ffmpeg.exe") + if not ppm.exists() or not ffmpeg: + raise RuntimeError("screenshot failed or ffmpeg.exe is not on PATH") + subprocess.run([ffmpeg, "-y", "-loglevel", "error", "-i", str(ppm), str(png)], check=True) + ppm.unlink(missing_ok=True) + log_event(logging.INFO, "screenshot", path=str(png)) + return {"path": str(png)} + + def dispatch(self, request: dict[str, object]) -> object: + op = request["op"] + if op == "status": + try: + self.monitor("info status") + running = True + except OSError: + running = False + return {"agent_connected": self.agent_ready.is_set(), "qemu_running": running, "log": str(LOG_PATH)} + if op == "start": return self.start() + if op == "stop": return self.stop() + if op == "ping": return self.ping() + if op == "exec": return self.exec(str(request["command"])) + if op == "put": return self.put(str(request["source"]), str(request["destination"])) + if op == "get": return self.get(str(request["source"]), str(request["destination"])) + if op == "screenshot": return self.screenshot() + if op == "monitor": return {"output": self.monitor(str(request["command"]))} + raise ValueError(f"unknown operation: {op}") + + +def serve_pipe(host: Host) -> None: + listener = Listener(PIPE, family="AF_PIPE") + log_event(logging.INFO, "control pipe ready", pipe=PIPE) + while True: + conn = listener.accept() + try: + request = conn.recv() + try: + conn.send({"ok": True, "result": host.dispatch(request)}) + except Exception as exc: + log_event(logging.ERROR, "control failed", error=str(exc)) + conn.send({"ok": False, "error": str(exc)}) + finally: + conn.close() + + +def main() -> None: + if os.name != "nt": + raise SystemExit("ferro-vm currently supports Windows only") + configure_logging() + host = Host() + threading.Thread(target=host.accept_agents, daemon=True).start() + serve_pipe(host) + + +if __name__ == "__main__": + main() diff --git a/tools/README.md b/tools/README.md index 9798245..2b0b4b8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,26 +1,48 @@ # Development tools -## QEMU console OCR +## Host support -Capture the current QEMU VGA screen and print detected console text: +Automation currently supports **Windows 10/11 only**. It requires `uv`, QEMU +with WHPX support, and `ffmpeg.exe` on `PATH`. The Python implementation uses +portable APIs where possible, but other hosts are not supported yet. + +## QEMU and FreeDOS automation + +Start the Python daemon and QEMU: ```powershell -uv run python tools/qemu_ocr.py +uv run ferro-vm start +uv run ferro-vm status ``` -Useful options: +`TCPAGENT.EXE` connects only to `127.0.0.1:5558`. Local commands use the +Windows named pipe `\\.\pipe\ferrolang-vm`; there is no controller or observer +TCP port. Monitor the append-only structured log in another terminal: ```powershell -# Machine-readable boxes, confidence scores, and text -uv run python tools/qemu_ocr.py --json +lnav .qemu/ferro-vm.log +``` -# OCR an existing screenshot without recapturing +Commands: + +```powershell +uv run ferro-vm ping +uv run ferro-vm exec 'dir C:\FEC' +uv run ferro-vm put fec/src/check.c 'C:\FEC\SRC\CHECK.C' +uv run ferro-vm get 'C:\FEC\TEST.OK' .qemu/TEST.OK +uv run ferro-vm screenshot +uv run ferro-vm ocr +uv run ferro-vm stop +``` + +The daemon logs command metadata, DOS output, exit status, transfers, and +agent lifecycle events as UTF-8 lines. It deliberately never logs raw binary +payloads or protocol hex. + +## Standalone OCR + +`tools/qemu_ocr.py` remains available for OCRing an existing image: + +```powershell uv run python tools/qemu_ocr.py --image .qemu/qemu-screen.png - -# Save extracted text -uv run python tools/qemu_ocr.py -o .qemu/qemu-screen.txt ``` - -The tool invokes `.qemu/screenshot.ps1`, runs RapidOCR with ONNX Runtime, sorts -recognized lines by screen position, and emits UTF-8 text. Screenshots and OCR -output under `.qemu/` remain ignored build artifacts. diff --git a/tools/dos_stage.py b/tools/dos_stage.py deleted file mode 100644 index 1218749..0000000 --- a/tools/dos_stage.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -"""Stage changed source files into the running FreeDOS VM over DOSAGENT. - -Usage: python tools/dos_stage.py host/file=C:\\FEC\\DEST.FILE [...] -The DOS agent uses a 115200-baud virtual COM1 link and accepts at most 4096 -binary payload bytes per ASCII-hex WRITE request. -""" -import hashlib -import socket -import sys -from pathlib import Path - -HOST = "127.0.0.1" -PORT = 5555 -DEFAULT_CHUNK_SIZE = 4096 - - -def request(line: str) -> str: - with socket.create_connection((HOST, PORT), timeout=30) as sock: - sock.sendall((line + "\n").encode("ascii")) - response = bytearray() - while not response.endswith(b"\n"): - part = sock.recv(65536) - if not part: - break - response.extend(part) - text = response.decode("ascii", "strict").strip() - if text.startswith("ERR "): - raise RuntimeError(bytes.fromhex(text[4:]).decode("utf-8", "replace")) - if not text.startswith("OK"): - raise RuntimeError("unexpected DOSAGENT response: " + text) - return text - - -def recv_line(sock: socket.socket) -> bytes: - line = bytearray() - while not line.endswith(b"\n"): - part = sock.recv(1) - if not part: - raise RuntimeError("TCP agent closed the connection") - line.extend(part) - return bytes(line).strip() - - -def fnv1a(data: bytes) -> int: - value = 2166136261 - for byte in data: - value = ((value ^ byte) * 16777619) & 0xFFFFFFFF - return value - - -def binary_stage(local: Path, remote: str) -> None: - data = local.read_bytes() - path_hex = remote.encode("ascii").hex().upper() - with socket.create_connection((HOST, PORT), timeout=30) as sock: - sock.sendall(("PUT %s %d\n" % (path_hex, len(data))).encode("ascii")) - sock.sendall(data) - response = recv_line(sock) - if not response.startswith(b"OK"): - raise RuntimeError("PUT failed: " + response.decode("ascii", "replace")) - sock.sendall(("HASH %s\n" % path_hex).encode("ascii")) - header = recv_line(sock).decode("ascii", "strict") - fields = header.split() - if len(fields) != 3 or fields[0] != "STAT": - raise RuntimeError("HASH failed: " + header) - remote_length = int(fields[1]) - remote_hash = int(fields[2], 16) - if remote_length != len(data) or remote_hash != fnv1a(data): - raise RuntimeError("verification mismatch for " + remote) - print("staged %s -> %s (%d bytes, sha256 %s, binary TCP)" % ( - local, remote, len(data), hashlib.sha256(data).hexdigest())) - - -def stage(local: Path, remote: str, chunk_size: int) -> None: - data = local.read_bytes() - remote_hex = remote.encode("ascii").hex().upper() - for offset in range(0, max(1, len(data)), chunk_size): - part = data[offset : offset + chunk_size] - mode = "T" if offset == 0 else "A" - request("WRITE %s %s %s" % (remote_hex, mode, part.hex().upper())) - remote_data = bytearray() - offset = 0 - while True: - response = request("READ %s %u" % (remote_hex, offset)) - _, eof, payload = response.split(" ", 2) - piece = bytes.fromhex(payload) - remote_data.extend(piece) - offset += len(piece) - if eof == "1": - break - if bytes(remote_data) != data: - raise RuntimeError("verification mismatch for " + remote) - print("staged %s -> %s (%d bytes, sha256 %s)" % ( - local, remote, len(data), hashlib.sha256(data).hexdigest())) - - -def main() -> int: - args = sys.argv[1:] - chunk_size = DEFAULT_CHUNK_SIZE - binary = False - if args and args[0] == "--binary": - binary = True - args = args[1:] - if len(args) >= 2 and args[0] == "--chunk-size": - chunk_size = int(args[1]) - args = args[2:] - if not args: - print("usage: dos_stage.py [--binary] [--chunk-size N] LOCAL=DOS_PATH [... ]", file=sys.stderr) - return 2 - for spec in args: - if "=" not in spec: - raise SystemExit("missing '=' in " + spec) - local, remote = spec.split("=", 1) - if binary: - binary_stage(Path(local), remote) - else: - stage(Path(local), remote, chunk_size) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/qemu_ocr.py b/tools/qemu_ocr.py index 41b3ee0..949380f 100644 --- a/tools/qemu_ocr.py +++ b/tools/qemu_ocr.py @@ -14,19 +14,11 @@ from rapidocr import RapidOCR ROOT = Path(__file__).resolve().parent.parent DEFAULT_IMAGE = ROOT / ".qemu" / "qemu-screen.png" -SCREENSHOT = ROOT / ".qemu" / "screenshot.ps1" def capture() -> Path: subprocess.run( - [ - "powershell", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-File", - str(SCREENSHOT), - ], + ["ferro-vm", "screenshot"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, diff --git a/tools/tcpagent/README.md b/tools/tcpagent/README.md index d0d60f0..afa3fff 100644 --- a/tools/tcpagent/README.md +++ b/tools/tcpagent/README.md @@ -1,8 +1,13 @@ # FreeDOS resident TCP agent `TCPAGENT.EXE` is a foreground resident automation process. It uses the mTCP -packet-driver stack and maintains an outbound connection to QEMU's host at -`10.0.2.2:5558`. The host relay keeps the existing Pi tool endpoint on 5555. +packet-driver stack and maintains an outbound connection to the QEMU host at +`10.0.2.2:5558`. + +The Windows-only Python `ferro-vm` daemon owns that listener. It logs metadata +and decoded command output to `.qemu/ferro-vm.log`; it does not expose an +observer/controller TCP port or emit binary payloads to the log. Local host +control uses a Windows named pipe. ## Build in FreeDOS @@ -17,16 +22,18 @@ therefore distributed under GPLv3 when linked with mTCP. ## Protocol -Legacy text commands remain for Pi tool compatibility: `PING`, `READ`, -`WRITE`, `LIST`, and `EXEC`. Fast staging uses binary framing: +`PING`, `READ`, `WRITE`, `LIST`, and `EXEC` use text commands. Fast transfer +commands are: - `PUT \n` -> `OK\r\n` - `GET \n` -> `DATA \r\n` - `HASH \n` -> `STAT \r\n` -Use `python tools/dos_stage.py --binary LOCAL=C:\\DOS\\PATH` for verified -staging. Verification reads only the length and FNV-1a hash, avoiding a slow -full-file return transfer. +The host invokes them through: -Use `.qemu/reset.ps1` to restart QEMU. It waits for an established TCP agent -and a successful PONG rather than sleeping for a fixed boot duration. +```powershell +uv run ferro-vm put host-file 'C:\DOS\FILE' +uv run ferro-vm get 'C:\DOS\FILE' host-file +``` + +No DOS-side change is required for the Python host automation. diff --git a/uv.lock b/uv.lock index faf4660..c8f60c6 100644 --- a/uv.lock +++ b/uv.lock @@ -172,7 +172,7 @@ wheels = [ [[package]] name = "ferrolang" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "onnxruntime" }, { name = "rapidocr" },