From 89e8785265021f34fca9ad808b19105aebb1f2a2 Mon Sep 17 00:00:00 2001 From: rawhide kobayashi Date: Thu, 27 Feb 2025 01:44:41 -0600 Subject: [PATCH] cleaned up serial code, added m - basic IP KVM achieved --- bios-maps/asrock/b650e-riptide-wifi.gv | 77 ++++++++++++++++++++++++++ esp32/src/main.cpp | 70 ++++++++++++++--------- platformio.ini | 6 +- webui/graphtest.py | 71 ++++++++++++++++++++++++ webui/ipkvm/__init__.py | 3 + webui/ipkvm/routes.py | 64 ++++++++++++--------- webui/ipkvm/static/mkb_handler.js | 51 ++++++++++++++++- webui/ipkvm/static/style.css | 2 +- webui/ipkvm/util/mkb.py | 50 ++++++++++++++++- webui/ipkvm/util/super_serial.py | 2 - webui/serial_test.py | 21 ++++++- 11 files changed, 351 insertions(+), 66 deletions(-) create mode 100644 bios-maps/asrock/b650e-riptide-wifi.gv create mode 100644 webui/graphtest.py delete mode 100644 webui/ipkvm/util/super_serial.py diff --git a/bios-maps/asrock/b650e-riptide-wifi.gv b/bios-maps/asrock/b650e-riptide-wifi.gv new file mode 100644 index 0000000..c93c354 --- /dev/null +++ b/bios-maps/asrock/b650e-riptide-wifi.gv @@ -0,0 +1,77 @@ +digraph bios +{ + //compound = true + node [shape = "box"] + style = dashed + color = red + main [label = "Main"] + //rankdir = "LR" + + oc_tweaker [label = "OC Tweaker"] + gaming_mode [label = "Gaming Mode" value = "Disabled"] + tdp_to_105w [label = "TDP to 105W" value = "Disabled"] + + subgraph cluster_advanced + { + advanced [label = "Advanced"] + cpu_config [label = "CPU Configuration"] + pci_config [label = "PCI Configuration"] + onb_dev_config [label = "Onboard Devices Configuration"] + stor_config [label = "Storage Configuration"] + acpi_config [label = "ACPI Configuration"] + usb_config [label = "USB Configuration"] + trust_com [label = "Trusted Computing"] + amd_cbs [label = "AMD CBS"] + amd_pbs [label = "AMD PBS"] + amd_oc [label = "AMD Overclocking"] + oc_menu [label = "OC Menu"] + } + + + + tool [label = "Tool"] + hw_mon [label = "H/W Monitor"] + security [label = "Security"] + boot [label = "Boot"] + exit [label = "Exit"] + save_and_exit [label = "Save Changes and Exit"] + + main -> oc_tweaker [keypath = "ArrowRight"] + oc_tweaker -> advanced [keypath = "ArrowRight"] + advanced -> tool [keypath = "ArrowRight"] + tool -> hw_mon [keypath = "ArrowRight"] + hw_mon -> security [keypath = "ArrowRight"] + security -> boot [keypath = "ArrowRight"] + boot -> exit [keypath = "ArrowRight"] + + exit -> boot [keypath = "ArrowLeft"] + boot -> security [keypath = "ArrowLeft"] + security -> hw_mon [keypath = "ArrowLeft"] + hw_mon -> tool [keypath = "ArrowLeft"] + tool -> advanced [keypath = "ArrowLeft"] + advanced -> oc_tweaker [keypath = "ArrowLeft"] + oc_tweaker -> main [keypath = "ArrowLeft"] + + oc_tweaker -> gaming_mode [keypath = "Home"] + gaming_mode -> oc_tweaker [keypath = "Home"] + gaming_mode -> tdp_to_105w [keypath = "ArrowDown"] + tdp_to_105w -> gaming_mode [keypath = "ArrowUp"] + + advanced -> cpu_config [keypath = "Home"] + cpu_config -> pci_config [keypath = "ArrowDown"] + pci_config -> onb_dev_config [keypath = "ArrowDown"] + onb_dev_config -> stor_config [keypath = "ArrowDown"] + stor_config -> acpi_config [keypath = "ArrowDown"] + acpi_config -> usb_config [keypath = "ArrowDown"] + usb_config -> trust_com [keypath = "ArrowDown"] + trust_com -> amd_cbs [keypath = "ArrowDown"] + amd_cbs -> amd_pbs [keypath = "ArrowDown"] + amd_pbs -> amd_oc [keypath = "ArrowDown"] + + cpu_config, pci_config, onb_dev_config, stor_config, acpi_config, usb_config, trust_com, amd_cbs, amd_pbs, amd_oc -> advanced [keypath = "Home"] + + amd_oc -> oc_menu [keypath = "Enter,Enter"] + oc_menu -> amd_oc [keypath = "Escape,Escape"] + + exit -> save_and_exit [keypath = "Enter,Enter"] +} diff --git a/esp32/src/main.cpp b/esp32/src/main.cpp index af27733..7fbdabb 100644 --- a/esp32/src/main.cpp +++ b/esp32/src/main.cpp @@ -12,7 +12,7 @@ void loop() {} #include #include -USBHIDMouse Mouse; +USBHIDAbsoluteMouse Mouse; USBHIDKeyboard Keyboard; HardwareSerial &host_serial = Serial; @@ -36,20 +36,11 @@ void setup() void loop() { - // Keyboard.write(0x4C); - // Keyboard.pressRaw(HID_KEY_DELETE); - // Keyboard.releaseRaw(HID_KEY_DELETE); - // put your main code here, to run repeatedly: - /*if (Serial.available() > 0) { - char inChar = Serial.read(); - - - }*/ - //while (mobo_serial.available()) - //{ - // char c = mobo_serial.read(); - // host_serial.write(c); - //} + while (mobo_serial.available()) + { + char c = mobo_serial.read(); + host_serial.write(c); + } if (host_serial.available()) { DeserializationError error = deserializeJson(mkb_input, host_serial); @@ -63,21 +54,46 @@ void loop() else { - JsonArray key_down = mkb_input["key_down"]; - JsonArray key_up = mkb_input["key_up"]; - //host_serial.println("Hej!"); - //serializeJsonPretty(key_down, host_serial); - //serializeJsonPretty(key_up, host_serial); - //host_serial.println("Hej2!"); - for (JsonVariant key : key_down) + //JsonArray key_down = mkb_input["key_down"]; + //JsonArray key_up = mkb_input["key_up"]; + ////host_serial.println("Hej!"); + ////serializeJsonPretty(key_down, host_serial); + ////serializeJsonPretty(key_up, host_serial); + ////host_serial.println("Hej2!"); + //for (JsonVariant key : key_down) + //{ + // Keyboard.pressRaw(key.as()); + // //host_serial.println(key.as()); + //} + //for (JsonVariant key : key_up) + //{ + // Keyboard.releaseRaw(key.as()); + // //host_serial.println(key.as()); + //} + + if (mkb_input["key_down"].is()) { - Keyboard.pressRaw(key.as()); - host_serial.println(key.as()); + Keyboard.pressRaw(mkb_input["key_down"].as()); } - for (JsonVariant key : key_up) + + else if (mkb_input["key_up"].is()) { - Keyboard.releaseRaw(key.as()); - host_serial.println(key.as()); + Keyboard.releaseRaw(mkb_input["key_up"].as()); + } + + else if (mkb_input["mouse_coord"].is()) + { + Mouse.move(mkb_input["mouse_coord"]["x"].as(), mkb_input["mouse_coord"]["y"].as()); + } + + else if (mkb_input["mouse_down"].is()) + { + Mouse.press(mkb_input["mouse_down"].as()); + } + + else if (mkb_input["mouse_up"].is()) + { + Mouse.release(mkb_input["mouse_up"].as()); } } } diff --git a/platformio.ini b/platformio.ini index 90ca7a3..fbf94c7 100644 --- a/platformio.ini +++ b/platformio.ini @@ -11,9 +11,9 @@ [common] -[env:rymcu-esp32-s3-devkitc-1] -platform = espressif32 -board = rymcu-esp32-s3-devkitc-1 +[env:esp32-s3-devkitc-1] +platform = https://github.com/Jason2866/platform-espressif32.git#Arduino/IDF53 +board = esp32-s3-devkitc-1 framework = arduino build_flags = -DARDUINO_USB_MODE=0 monitor_speed = 115200 diff --git a/webui/graphtest.py b/webui/graphtest.py new file mode 100644 index 0000000..bf4adee --- /dev/null +++ b/webui/graphtest.py @@ -0,0 +1,71 @@ +import networkx as nx +import serial +from ipkvm.util.mkb import HIDKeyCode +import json +import time + +# Load the Graphviz file +graph = nx.nx_agraph.read_dot("bios-maps/asrock/b650e-riptide-wifi.gv") + +print(graph) + +print(graph.edges()) + +# Example: Access node attributes +for node, data in graph.nodes(data=True): + print(f"Node: {node}, Attributes: {data}") + +print(graph.edges(data=True)) + +# Example: Access edge attributes (keypress actions) +for edge_a, edge_b, data in graph.edges(data=True): + print(f"Edge: {edge_a} to {edge_b}, Attributes: {data}") + +path = nx.shortest_path(graph, "exit", "oc_menu") + +for pair in nx.utils.pairwise(path): + print(pair) + print(graph.edges(pair, data=True)) + +edge_path = list(zip(path[:-1], path[1:])) + +print("Node path:", path) +print("Edge path:", edge_path) + +edge_path_with_data = [(u, v, graph[u][v]) for u, v in edge_path] +print("Edge path with data:", edge_path_with_data) + +print("GENERATOR TEST") + +for path in sorted(nx.all_simple_edge_paths(graph, "exit", "oc_menu")): + for edge in path: + print(edge) + keys = graph.get_edge_data(edge[0], edge[1])[0]["keypath"].split(',') + print(keys) + + with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser: + for key in keys: + test_json_a = { + "mouseX": 99999, + "mouseY": 99999, + "mouse_down": ["rbutton", "lbutton"], + "mouse_up": ["otherbutton"], + "key_down": [HIDKeyCode[key]], + "key_up": [] + } +# + print(HIDKeyCode[key]) + ser.write(json.dumps(test_json_a).encode()) + + test_json_a = { + "mouseX": 99999, + "mouseY": 99999, + "mouse_down": ["rbutton", "lbutton"], + "mouse_up": ["otherbutton"], + "key_down": [], + "key_up": [HIDKeyCode[key]] + } +# + print(HIDKeyCode[key]) + ser.write(json.dumps(test_json_a).encode()) + #time.sleep(0.1) \ No newline at end of file diff --git a/webui/ipkvm/__init__.py b/webui/ipkvm/__init__.py index 5eb67cb..55c9b8b 100644 --- a/webui/ipkvm/__init__.py +++ b/webui/ipkvm/__init__.py @@ -4,6 +4,7 @@ from flask_socketio import SocketIO import json import logging + app = Flask(__name__) ui = SocketIO(app) logger = app.logger @@ -94,7 +95,9 @@ elif len(listdir("profiles")) == 1: from ipkvm.util import video from ipkvm import feed +from ipkvm.util.mkb import Esp32Serial frame_buffer = feed.FrameBuffer() +esp32_serial = Esp32Serial() from ipkvm import routes diff --git a/webui/ipkvm/routes.py b/webui/ipkvm/routes.py index 355e199..452c442 100644 --- a/webui/ipkvm/routes.py +++ b/webui/ipkvm/routes.py @@ -1,10 +1,7 @@ from ipkvm import app, ui -from ipkvm import frame_buffer +from ipkvm import frame_buffer, esp32_serial from flask import Response, render_template -import time -from ipkvm.util.mkb import HIDKeyCode -import serial -import json +from ipkvm.util.mkb import HIDKeyCode, HIDMouseScanCodes def generate_frames(): while True: @@ -14,34 +11,47 @@ def generate_frames(): b'Content-Type: image/jpeg\r\n\r\n' + frame_buffer.cur_frame + b'\r\n') @ui.on('key_down') -def handle_keydown(data): - test_json_a = { - "mouseX": 99999, - "mouseY": 99999, - "mouse_down": ["rbutton", "lbutton"], - "mouse_up": ["otherbutton"], - "key_up": [], - "key_down": [HIDKeyCode[data]] +def handle_keydown(data: str): + msg = { + "key_down": HIDKeyCode[data].value } - print(HIDKeyCode[data]) - with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser: - ser.write(json.dumps(test_json_a).encode()) + esp32_serial.mkb_queue.put(msg) @ui.on('key_up') -def handle_keyup(data): - test_json_a = { - "mouseX": 99999, - "mouseY": 99999, - "mouse_down": ["rbutton", "lbutton"], - "mouse_up": ["otherbutton"], - "key_up": [HIDKeyCode[data]], - "key_down": [] +def handle_keyup(data: str): + msg = { + "key_up": HIDKeyCode[data].value } - print(HIDKeyCode[data]) - with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser: - ser.write(json.dumps(test_json_a).encode()) + esp32_serial.mkb_queue.put(msg) + +@ui.on("mouse_move") +def handle_mousemove(data: list[int]): + msg = { + "mouse_coord": { + "x": data[0], + "y": data[1] + } + } + + esp32_serial.mkb_queue.put(msg) + +@ui.on('mouse_down') +def handle_mousedown(data: int): + msg = { + "mouse_down": HIDMouseScanCodes[data] + } + + esp32_serial.mkb_queue.put(msg) + +@ui.on('mouse_up') +def handle_mouseup(data: int): + msg = { + "mouse_up": HIDMouseScanCodes[data] + } + + esp32_serial.mkb_queue.put(msg) @app.route('/video_feed') def video_feed(): diff --git a/webui/ipkvm/static/mkb_handler.js b/webui/ipkvm/static/mkb_handler.js index d65f2e8..8eaa754 100644 --- a/webui/ipkvm/static/mkb_handler.js +++ b/webui/ipkvm/static/mkb_handler.js @@ -1,29 +1,76 @@ const streamview = document.getElementById('streamview'); +var last_mouse_update = Date.now() var socket = io(); function keydown_handler(event) { - console.log(`Key pressed: ${event.code}`); socket.emit('key_down', event.code); + event.preventDefault(); + event.stopPropagation(); } function keyup_handler(event) { - console.log(`Key released: ${event.code}`); socket.emit('key_up', event.code); + event.preventDefault(); + event.stopPropagation(); +} + +function mousemove_handler(event) +{ + // limit the mouse update rate to 60fps (approximately) because serial bandwidth limits be low + if (Date.now() - last_mouse_update >= 16) + { + last_mouse_update = Date.now(); + const bounds = streamview.getBoundingClientRect(); + const x = event.clientX - bounds.left; + const y = event.clientY - bounds.top; + const x_scale_factor = 32768 / streamview.clientWidth; + const y_scale_factor = 32768 / streamview.clientHeight; + const scaled_x = x * x_scale_factor; + const scaled_y = y * y_scale_factor + socket.emit("mouse_move", [scaled_x, scaled_y]); + } +} + +function mousedown_handler(event) +{ + socket.emit('mouse_down', event.button); + event.preventDefault(); + event.stopPropagation(); +} + +function mouseup_handler(event) +{ + socket.emit('mouse_up', event.button); + event.preventDefault(); + event.stopPropagation(); +} + +function prevent_right_click(event) +{ + event.preventDefault(); } function enable_listener() { document.addEventListener("keydown", keydown_handler); document.addEventListener("keyup", keyup_handler); + document.addEventListener("mousemove", mousemove_handler); + document.addEventListener("mousedown", mousedown_handler); + document.addEventListener("mouseup", mouseup_handler); + document.addEventListener('contextmenu', prevent_right_click); } function disable_listener() { document.removeEventListener("keydown", keydown_handler); document.removeEventListener("keyup", keyup_handler); + document.removeEventListener("mousemove", mousemove_handler); + document.removeEventListener("mousedown", mousedown_handler); + document.removeEventListener("mouseup", mouseup_handler); + document.removeEventListener('contextmenu', prevent_right_click); } streamview.addEventListener("mouseenter", enable_listener); diff --git a/webui/ipkvm/static/style.css b/webui/ipkvm/static/style.css index 73aad32..807ef23 100644 --- a/webui/ipkvm/static/style.css +++ b/webui/ipkvm/static/style.css @@ -1,6 +1,6 @@ .stream-container { margin: 0 auto; - padding: 20px; + border: 20px; } .stream-view { width: 100%; diff --git a/webui/ipkvm/util/mkb.py b/webui/ipkvm/util/mkb.py index 05f6e50..ed9a979 100644 --- a/webui/ipkvm/util/mkb.py +++ b/webui/ipkvm/util/mkb.py @@ -1,9 +1,24 @@ from enum import IntEnum +from os import name +import serial +from ipkvm import profile +import threading +from queue import Queue +import json + +# Python can't make number to number enums??? +HIDMouseScanCodes = { + 0: 1, + 2: 2, + 1: 4, + 3: 8, + 4: 16 +} # God Bless CHADGPT class HIDKeyCode(IntEnum): """ - Enum that translates modern JS key.code values to HID scancodes. + Enum that translates modern JS key.code andvalues to HID scancodes. """ # Letter keys (A-Z) KeyA = 4 @@ -132,3 +147,36 @@ class HIDKeyCode(IntEnum): ShiftRight = 229 AltRight = 230 MetaRight = 231 # Windows / Command key (right) + +class Esp32Serial(threading.Thread): + def __init__(self): + super().__init__() + self.post_code_queue: Queue[str] = Queue() + self.mkb_queue: Queue[dict[str, int | dict[str, int]]] = Queue() + self.change_serial_device = threading.Event() + self.device = self.get_device() + + self.start() + + def run(self): + while True: + if self.change_serial_device.is_set(): + self.change_serial_device.clear() + self.device = self.get_device() + + with self.device as ser: + while not self.mkb_queue.empty(): + msg = self.mkb_queue.get() + ser.write(json.dumps(msg).encode()) + + while ser.in_waiting > 0: + print(ser.read().hex()) + # self.post_code_queue.put(ser.read().hex()) + + def get_device(self): + if name == "posix": + return serial.Serial(f"/dev/serial/by-id/{profile["esp32_serial"]}", 115200, bytesize=serial.EIGHTBITS, + parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) + + else: + raise RuntimeError("Your OS is unsupported!") diff --git a/webui/ipkvm/util/super_serial.py b/webui/ipkvm/util/super_serial.py deleted file mode 100644 index 28480f2..0000000 --- a/webui/ipkvm/util/super_serial.py +++ /dev/null @@ -1,2 +0,0 @@ -import serial - diff --git a/webui/serial_test.py b/webui/serial_test.py index 1a012e2..0de0c74 100644 --- a/webui/serial_test.py +++ b/webui/serial_test.py @@ -22,6 +22,20 @@ test_json_b = { "key_down": [] } +test_json_c = { + "mouse_coord": { + "x": 100, + "y": 100, + } +} + +test_json_d = { + "mouse_coord": { + "x": 32000, + "y": 32000, + } +} + def read_serial(port): try: # Open the serial port @@ -30,11 +44,12 @@ def read_serial(port): while True: # Read a line from the serial port while ser.in_waiting > 0: - line = ser.readline() + line = str(ser.read().hex()) print(f'{datetime.datetime.now()} {line}') # Print the raw data - ser.write(json.dumps(test_json_a).encode()) - ser.write(json.dumps(test_json_b).encode()) + ser.write(json.dumps(test_json_c).encode()) + time.sleep(1) + ser.write(json.dumps(test_json_d).encode()) time.sleep(1) except serial.SerialException as e: print(f"Error: {e}")