cleaned up serial code, added m - basic IP KVM achieved

bios-maps/asrock/b650e-riptide-wifi.gv

@@ -0,0 +1,77 @@
+digraph bios
+{
+    //compound = true
+    node [shape = "box"]
+    style = dashed
+    color = red
+    main [label = "Main"]
+    //rankdir = "LR"
+
+    oc_tweaker [label = "OC Tweaker"]
+    gaming_mode [label = "Gaming Mode" value = "Disabled"]
+    tdp_to_105w [label = "TDP to 105W" value = "Disabled"]
+
+    subgraph cluster_advanced
+    {
+        advanced [label = "Advanced"]
+        cpu_config [label = "CPU Configuration"]
+        pci_config [label = "PCI Configuration"]
+        onb_dev_config [label = "Onboard Devices Configuration"]
+        stor_config [label = "Storage Configuration"]
+        acpi_config [label = "ACPI Configuration"]
+        usb_config [label = "USB Configuration"]
+        trust_com [label = "Trusted Computing"]
+        amd_cbs [label = "AMD CBS"]
+        amd_pbs [label = "AMD PBS"]
+        amd_oc [label = "AMD Overclocking"]
+        oc_menu [label = "OC Menu"]
+    }
+
+    
+
+    tool [label = "Tool"]
+    hw_mon [label = "H/W Monitor"]
+    security [label = "Security"]
+    boot [label = "Boot"]
+    exit [label = "Exit"]
+    save_and_exit [label = "Save Changes and Exit"]
+
+    main -> oc_tweaker [keypath = "ArrowRight"]
+    oc_tweaker -> advanced [keypath = "ArrowRight"]
+    advanced -> tool [keypath = "ArrowRight"]
+    tool -> hw_mon [keypath = "ArrowRight"]
+    hw_mon -> security [keypath = "ArrowRight"]
+    security -> boot [keypath = "ArrowRight"]
+    boot -> exit [keypath = "ArrowRight"]
+
+    exit -> boot [keypath = "ArrowLeft"]
+    boot -> security [keypath = "ArrowLeft"]
+    security -> hw_mon [keypath = "ArrowLeft"]
+    hw_mon -> tool [keypath = "ArrowLeft"]
+    tool -> advanced [keypath = "ArrowLeft"]
+    advanced -> oc_tweaker [keypath = "ArrowLeft"]
+    oc_tweaker -> main [keypath = "ArrowLeft"]
+
+    oc_tweaker -> gaming_mode [keypath = "Home"]
+    gaming_mode -> oc_tweaker [keypath = "Home"]
+    gaming_mode -> tdp_to_105w [keypath = "ArrowDown"]
+    tdp_to_105w -> gaming_mode [keypath = "ArrowUp"]
+
+    advanced -> cpu_config [keypath = "Home"]
+    cpu_config -> pci_config [keypath = "ArrowDown"]
+    pci_config -> onb_dev_config [keypath = "ArrowDown"]
+    onb_dev_config -> stor_config [keypath = "ArrowDown"]
+    stor_config -> acpi_config [keypath = "ArrowDown"]
+    acpi_config -> usb_config [keypath = "ArrowDown"]
+    usb_config -> trust_com [keypath = "ArrowDown"]
+    trust_com -> amd_cbs [keypath = "ArrowDown"]
+    amd_cbs -> amd_pbs [keypath = "ArrowDown"]
+    amd_pbs -> amd_oc [keypath = "ArrowDown"]
+
+    cpu_config, pci_config, onb_dev_config, stor_config, acpi_config, usb_config, trust_com, amd_cbs, amd_pbs, amd_oc -> advanced [keypath = "Home"]
+
+    amd_oc -> oc_menu [keypath = "Enter,Enter"]
+    oc_menu -> amd_oc [keypath = "Escape,Escape"]
+
+    exit -> save_and_exit [keypath = "Enter,Enter"]
+}

esp32/src/main.cpp

@@ -12,7 +12,7 @@ void loop() {}
 #include <USBHIDMouse.h>
 #include <USBHIDKeyboard.h>
 
-USBHIDMouse Mouse;
+USBHIDAbsoluteMouse Mouse;
 USBHIDKeyboard Keyboard;
 
 HardwareSerial &host_serial = Serial;
@@ -36,20 +36,11 @@ void setup()
 
 void loop()
 {
-    // Keyboard.write(0x4C);
+    while (mobo_serial.available())
-    // Keyboard.pressRaw(HID_KEY_DELETE);
+    {
-    // Keyboard.releaseRaw(HID_KEY_DELETE);
+        char c = mobo_serial.read();
-    //  put your main code here, to run repeatedly:
+        host_serial.write(c);
-    /*if (Serial.available() > 0) {
+    }
-      char inChar = Serial.read();
-
-
-    }*/
-    //while (mobo_serial.available())
-    //{
-    //    char c = mobo_serial.read();
-    //    host_serial.write(c);
-    //}
     if (host_serial.available())
     {
         DeserializationError error = deserializeJson(mkb_input, host_serial);
@@ -63,21 +54,46 @@ void loop()
 
         else
         {
-            JsonArray key_down = mkb_input["key_down"];
+            //JsonArray key_down = mkb_input["key_down"];
-            JsonArray key_up = mkb_input["key_up"];
+            //JsonArray key_up = mkb_input["key_up"];
-            //host_serial.println("Hej!");
+            ////host_serial.println("Hej!");
-            //serializeJsonPretty(key_down, host_serial);
+            ////serializeJsonPretty(key_down, host_serial);
-            //serializeJsonPretty(key_up, host_serial);
+            ////serializeJsonPretty(key_up, host_serial);
-            //host_serial.println("Hej2!");
+            ////host_serial.println("Hej2!");
-            for (JsonVariant key : key_down)
+            //for (JsonVariant key : key_down)
+            //{
+            //    Keyboard.pressRaw(key.as<u8_t>());
+            //    //host_serial.println(key.as<u8_t>());
+            //}
+            //for (JsonVariant key : key_up)
+            //{
+            //    Keyboard.releaseRaw(key.as<u8_t>());
+            //    //host_serial.println(key.as<u8_t>());
+            //}
+
+            if (mkb_input["key_down"].is<JsonVariant>())
             {
-                Keyboard.pressRaw(key.as<u8_t>());
+                Keyboard.pressRaw(mkb_input["key_down"].as<uint8_t>());
-                host_serial.println(key.as<u8_t>());
             }
-            for (JsonVariant key : key_up)
+
+            else if (mkb_input["key_up"].is<JsonVariant>())
             {
-                Keyboard.releaseRaw(key.as<u8_t>());
+                Keyboard.releaseRaw(mkb_input["key_up"].as<uint8_t>());
-                host_serial.println(key.as<u8_t>());
+            }
+
+            else if (mkb_input["mouse_coord"].is<JsonVariant>())
+            {
+                Mouse.move(mkb_input["mouse_coord"]["x"].as<int16_t>(), mkb_input["mouse_coord"]["y"].as<int16_t>());
+            }
+
+            else if (mkb_input["mouse_down"].is<JsonVariant>())
+            {
+                Mouse.press(mkb_input["mouse_down"].as<uint8_t>());
+            }
+
+            else if (mkb_input["mouse_up"].is<JsonVariant>())
+            {
+                Mouse.release(mkb_input["mouse_up"].as<uint8_t>());
             }
         }
     }

platformio.ini

@@ -11,9 +11,9 @@
 [common]
 
 
-[env:rymcu-esp32-s3-devkitc-1]
+[env:esp32-s3-devkitc-1]
-platform = espressif32
+platform = https://github.com/Jason2866/platform-espressif32.git#Arduino/IDF53
-board = rymcu-esp32-s3-devkitc-1
+board = esp32-s3-devkitc-1
 framework = arduino
 build_flags = -DARDUINO_USB_MODE=0
 monitor_speed = 115200

webui/graphtest.py

@@ -0,0 +1,71 @@
+import networkx as nx
+import serial
+from ipkvm.util.mkb import HIDKeyCode
+import json
+import time
+
+# Load the Graphviz file
+graph = nx.nx_agraph.read_dot("bios-maps/asrock/b650e-riptide-wifi.gv")
+
+print(graph)
+
+print(graph.edges())
+
+# Example: Access node attributes
+for node, data in graph.nodes(data=True):
+    print(f"Node: {node}, Attributes: {data}")
+
+print(graph.edges(data=True))
+
+# Example: Access edge attributes (keypress actions)
+for edge_a, edge_b, data in graph.edges(data=True):
+    print(f"Edge: {edge_a} to {edge_b}, Attributes: {data}")
+
+path = nx.shortest_path(graph, "exit", "oc_menu")
+
+for pair in nx.utils.pairwise(path):
+    print(pair)
+    print(graph.edges(pair, data=True))
+
+edge_path = list(zip(path[:-1], path[1:]))
+
+print("Node path:", path)
+print("Edge path:", edge_path)
+
+edge_path_with_data = [(u, v, graph[u][v]) for u, v in edge_path]
+print("Edge path with data:", edge_path_with_data)
+
+print("GENERATOR TEST")
+
+for path in sorted(nx.all_simple_edge_paths(graph, "exit", "oc_menu")):
+    for edge in path:
+        print(edge)
+        keys = graph.get_edge_data(edge[0], edge[1])[0]["keypath"].split(',')
+        print(keys)
+
+        with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+            for key in keys:
+                test_json_a = {
+                    "mouseX": 99999,
+                    "mouseY": 99999,
+                    "mouse_down": ["rbutton", "lbutton"],
+                    "mouse_up": ["otherbutton"],
+                    "key_down": [HIDKeyCode[key]],
+                    "key_up": []
+                }
+#       
+                print(HIDKeyCode[key])
+                ser.write(json.dumps(test_json_a).encode())
+
+                test_json_a = {
+                    "mouseX": 99999,
+                    "mouseY": 99999,
+                    "mouse_down": ["rbutton", "lbutton"],
+                    "mouse_up": ["otherbutton"],
+                    "key_down": [],
+                    "key_up": [HIDKeyCode[key]]
+                }
+#       
+                print(HIDKeyCode[key])
+                ser.write(json.dumps(test_json_a).encode())
+                #time.sleep(0.1)

webui/ipkvm/__init__.py

@@ -4,6 +4,7 @@ from flask_socketio import SocketIO
 import json
 import logging
 
+
 app = Flask(__name__)
 ui = SocketIO(app)
 logger = app.logger
@@ -94,7 +95,9 @@ elif len(listdir("profiles")) == 1:
 
 from ipkvm.util import video
 from ipkvm import feed
+from ipkvm.util.mkb import Esp32Serial
 
 frame_buffer = feed.FrameBuffer()
+esp32_serial = Esp32Serial()
 
 from ipkvm import routes

webui/ipkvm/routes.py

@@ -1,10 +1,7 @@
 from ipkvm import app, ui
-from ipkvm import frame_buffer
+from ipkvm import frame_buffer, esp32_serial
 from flask import Response, render_template
-import time
+from ipkvm.util.mkb import HIDKeyCode, HIDMouseScanCodes
-from ipkvm.util.mkb import HIDKeyCode
-import serial
-import json
 
 def generate_frames():
     while True:
@@ -14,34 +11,47 @@ def generate_frames():
                b'Content-Type: image/jpeg\r\n\r\n' + frame_buffer.cur_frame + b'\r\n')
         
 @ui.on('key_down')
-def handle_keydown(data):
+def handle_keydown(data: str):
-    test_json_a = {
+    msg = {
-      "mouseX": 99999,
+      "key_down": HIDKeyCode[data].value
-      "mouseY": 99999,
-      "mouse_down": ["rbutton", "lbutton"],
-      "mouse_up": ["otherbutton"],
-      "key_up": [],
-      "key_down": [HIDKeyCode[data]]
     }
 
-    print(HIDKeyCode[data])
+    esp32_serial.mkb_queue.put(msg)
-    with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
-        ser.write(json.dumps(test_json_a).encode())
 
 @ui.on('key_up')
-def handle_keyup(data):
+def handle_keyup(data: str):
-    test_json_a = {
+    msg = {
-      "mouseX": 99999,
+      "key_up": HIDKeyCode[data].value
-      "mouseY": 99999,
-      "mouse_down": ["rbutton", "lbutton"],
-      "mouse_up": ["otherbutton"],
-      "key_up": [HIDKeyCode[data]],
-      "key_down": []
     }
 
-    print(HIDKeyCode[data])
+    esp32_serial.mkb_queue.put(msg)
-    with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+
-        ser.write(json.dumps(test_json_a).encode())
+@ui.on("mouse_move")
+def handle_mousemove(data: list[int]):
+    msg = {
+      "mouse_coord": {
+          "x": data[0],
+          "y": data[1]
+      }
+    }
+
+    esp32_serial.mkb_queue.put(msg)
+
+@ui.on('mouse_down')
+def handle_mousedown(data: int):
+    msg = {
+      "mouse_down": HIDMouseScanCodes[data]
+    }
+    
+    esp32_serial.mkb_queue.put(msg)
+
+@ui.on('mouse_up')
+def handle_mouseup(data: int):
+    msg = {
+      "mouse_up": HIDMouseScanCodes[data]
+    }
+    
+    esp32_serial.mkb_queue.put(msg)
 
 @app.route('/video_feed')
 def video_feed():

webui/ipkvm/static/mkb_handler.js

@@ -1,29 +1,76 @@
 const streamview = document.getElementById('streamview');
+var last_mouse_update = Date.now()
 
 var socket = io();
 
 function keydown_handler(event)
 {
-    console.log(`Key pressed: ${event.code}`);
     socket.emit('key_down', event.code);
+    event.preventDefault();
+    event.stopPropagation();
 }
 
 function keyup_handler(event)
 {
-    console.log(`Key released: ${event.code}`);
     socket.emit('key_up', event.code);
+    event.preventDefault();
+    event.stopPropagation();
+}
+
+function mousemove_handler(event)
+{
+    // limit the mouse update rate to 60fps (approximately) because serial bandwidth limits be low
+    if (Date.now() - last_mouse_update >= 16)
+    {
+        last_mouse_update = Date.now();
+        const bounds = streamview.getBoundingClientRect();
+        const x = event.clientX - bounds.left;
+        const y = event.clientY - bounds.top;
+        const x_scale_factor = 32768 / streamview.clientWidth;
+        const y_scale_factor = 32768 / streamview.clientHeight;
+        const scaled_x = x * x_scale_factor;
+        const scaled_y = y * y_scale_factor
+        socket.emit("mouse_move", [scaled_x, scaled_y]);
+    }
+}
+
+function mousedown_handler(event)
+{
+    socket.emit('mouse_down', event.button);
+    event.preventDefault();
+    event.stopPropagation();
+}
+
+function mouseup_handler(event)
+{
+    socket.emit('mouse_up', event.button);
+    event.preventDefault();
+    event.stopPropagation();
+}
+
+function prevent_right_click(event)
+{
+    event.preventDefault();
 }
 
 function enable_listener()
 {
     document.addEventListener("keydown", keydown_handler);
     document.addEventListener("keyup", keyup_handler);
+    document.addEventListener("mousemove", mousemove_handler);
+    document.addEventListener("mousedown", mousedown_handler);
+    document.addEventListener("mouseup", mouseup_handler);
+    document.addEventListener('contextmenu', prevent_right_click);
 }
 
 function disable_listener()
 {
     document.removeEventListener("keydown", keydown_handler);
     document.removeEventListener("keyup", keyup_handler);
+    document.removeEventListener("mousemove", mousemove_handler);
+    document.removeEventListener("mousedown", mousedown_handler);
+    document.removeEventListener("mouseup", mouseup_handler);
+    document.removeEventListener('contextmenu', prevent_right_click);
 }
 
 streamview.addEventListener("mouseenter", enable_listener);

webui/ipkvm/static/style.css

@@ -1,6 +1,6 @@
 .stream-container {
     margin: 0 auto;
-    padding: 20px;
+    border: 20px;
 }
 .stream-view {
     width: 100%;

webui/ipkvm/util/mkb.py

@@ -1,9 +1,24 @@
 from enum import IntEnum
+from os import name
+import serial
+from ipkvm import profile
+import threading
+from queue import Queue
+import json
+
+# Python can't make number to number enums???
+HIDMouseScanCodes = {
+    0: 1,
+    2: 2,
+    1: 4,
+    3: 8,
+    4: 16
+}
 
 # God Bless CHADGPT
 class HIDKeyCode(IntEnum):
     """
-    Enum that translates modern JS key.code values to HID scancodes.
+    Enum that translates modern JS key.code andvalues to HID scancodes.
     """
     # Letter keys (A-Z)
     KeyA = 4
@@ -132,3 +147,36 @@ class HIDKeyCode(IntEnum):
     ShiftRight = 229
     AltRight = 230
     MetaRight = 231  # Windows / Command key (right)
+
+class Esp32Serial(threading.Thread):
+    def __init__(self):
+        super().__init__()
+        self.post_code_queue: Queue[str] = Queue()
+        self.mkb_queue: Queue[dict[str, int | dict[str, int]]] = Queue()
+        self.change_serial_device = threading.Event()
+        self.device = self.get_device()
+
+        self.start()
+
+    def run(self):
+        while True:
+            if self.change_serial_device.is_set():
+                self.change_serial_device.clear()
+                self.device = self.get_device()
+            
+            with self.device as ser:
+                while not self.mkb_queue.empty():
+                    msg = self.mkb_queue.get()
+                    ser.write(json.dumps(msg).encode())
+
+                while ser.in_waiting > 0:
+                    print(ser.read().hex())
+                    # self.post_code_queue.put(ser.read().hex())
+
+    def get_device(self):
+        if name == "posix":
+            return serial.Serial(f"/dev/serial/by-id/{profile["esp32_serial"]}", 115200, bytesize=serial.EIGHTBITS,
+                                 parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE)
+        
+        else:
+            raise RuntimeError("Your OS is unsupported!")

webui/ipkvm/util/super_serial.py

@@ -1,2 +0,0 @@
-import serial
-

webui/serial_test.py

@@ -22,6 +22,20 @@ test_json_b = {
   "key_down": []
 }
 
+test_json_c = {
+    "mouse_coord": {
+        "x": 100,
+        "y": 100,
+    }
+}
+
+test_json_d = {
+    "mouse_coord": {
+        "x": 32000,
+        "y": 32000,
+    }
+}
+
 def read_serial(port):
     try:
         # Open the serial port
@@ -30,11 +44,12 @@ def read_serial(port):
             while True:
                 # Read a line from the serial port
                 while ser.in_waiting > 0:
-                    line = ser.readline()   
+                    line = str(ser.read().hex())
                     print(f'{datetime.datetime.now()} {line}')
                 # Print the raw data
-                ser.write(json.dumps(test_json_a).encode())
+                ser.write(json.dumps(test_json_c).encode())
-                ser.write(json.dumps(test_json_b).encode())
+                time.sleep(1)
+                ser.write(json.dumps(test_json_d).encode())
                 time.sleep(1)
     except serial.SerialException as e:
         print(f"Error: {e}")