'k' and 'v' are minimally functional

.gitignore

@@ -4,3 +4,4 @@
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/ipch
+ArduinoJson/

esp32/src/main.cpp

@@ -7,40 +7,86 @@ void loop() {}
 #else
 
 #include <Arduino.h>
+#include <ArduinoJson.h>
 #include <USB.h>
 #include <USBHIDMouse.h>
 #include <USBHIDKeyboard.h>
+
 USBHIDMouse Mouse;
 USBHIDKeyboard Keyboard;
 
+HardwareSerial &host_serial = Serial;
+HardwareSerial &mobo_serial = Serial1;
+
+JsonDocument mkb_input;
+
 // put function declarations here:
 int myFunction(int, int);
 
-void setup() {
+char buffer[100];
-  // put your setup code here, to run once:
+
-  int result = myFunction(2, 3);
+void setup()
-  Serial.begin(115200);
+{
-  Mouse.begin();
+    host_serial.begin(115200);
-  Keyboard.begin();
+    mobo_serial.begin(115200, SERIAL_8N1, 18);
-  USB.begin();
+    Mouse.begin();
+    Keyboard.begin();
+    USB.begin();
 }
 
-void loop() {
+void loop()
-  //Keyboard.write(0x4C);
+{
-  //Keyboard.pressRaw(HID_KEY_DELETE);
+    // Keyboard.write(0x4C);
-  //Keyboard.releaseRaw(HID_KEY_DELETE);
+    // Keyboard.pressRaw(HID_KEY_DELETE);
-  // put your main code here, to run repeatedly:
+    // Keyboard.releaseRaw(HID_KEY_DELETE);
-  /*if (Serial.available() > 0) {
+    //  put your main code here, to run repeatedly:
-    char inChar = Serial.read();
+    /*if (Serial.available() > 0) {
+      char inChar = Serial.read();
 
-    
+
-  }*/
+    }*/
- sleep(1000);
+    //while (mobo_serial.available())
+    //{
+    //    char c = mobo_serial.read();
+    //    host_serial.write(c);
+    //}
+    if (host_serial.available())
+    {
+        DeserializationError error = deserializeJson(mkb_input, host_serial);
+
+        if (error)
+        {
+            host_serial.print("deserializeJson() failed: ");
+            host_serial.println(error.c_str());
+            return;
+        }
+
+        else
+        {
+            JsonArray key_down = mkb_input["key_down"];
+            JsonArray key_up = mkb_input["key_up"];
+            //host_serial.println("Hej!");
+            //serializeJsonPretty(key_down, host_serial);
+            //serializeJsonPretty(key_up, host_serial);
+            //host_serial.println("Hej2!");
+            for (JsonVariant key : key_down)
+            {
+                Keyboard.pressRaw(key.as<u8_t>());
+                host_serial.println(key.as<u8_t>());
+            }
+            for (JsonVariant key : key_up)
+            {
+                Keyboard.releaseRaw(key.as<u8_t>());
+                host_serial.println(key.as<u8_t>());
+            }
+        }
+    }
 }
 
 // put function definitions here:
-int myFunction(int x, int y) {
+int myFunction(int x, int y)
-  return x + y;
+{
+    return x + y;
 }
 
 #endif /* ARDUINO_USB_MODE */

platformio.ini

@@ -17,6 +17,7 @@ board = rymcu-esp32-s3-devkitc-1
 framework = arduino
 build_flags = -DARDUINO_USB_MODE=0
 monitor_speed = 115200
+lib_deps = bblanchon/ArduinoJson @ ~7.3.0
 
 [platformio]
 src_dir = esp32/src

profiles/hong.json

@@ -1 +1 @@
-{"video_device": {"friendly_name": "WARRKY USB 3.0", "format": "mjpeg", "resolution": "1920x1080", "fps": "60.000"}, "esp32_serial": "usb-Espressif_USB_JTAG_serial_debug_unit_CC:8D:A2:0F:C0:08-if00"}
+{"video_device": {"friendly_name": "WARRKY USB 3.0", "format": "mjpeg", "resolution": "1920x1080", "fps": "60.000"}, "esp32_serial": "usb-1a86_USB_Single_Serial_585D015807-if00"}

serialtest.py

@@ -0,0 +1,28 @@
+import serial
+import sys
+import datetime
+
+def read_serial(port):
+    try:
+        # Open the serial port
+        with serial.Serial(port, 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+            print(f"Listening on {port} at 115200 baud...")
+            while True:
+                # Read a line from the serial port
+                line = ser.read()   
+                # Print the raw data
+                print(f'{datetime.datetime.now()} {line}')
+    except serial.SerialException as e:
+        print(f"Error: {e}")
+    except KeyboardInterrupt:
+        print("Exiting...")
+        sys.exit()
+
+if __name__ == "__main__":
+    #if len(sys.argv) != 2:
+    #    print("Usage: python read_serial.py <port>")
+    #    print("Example: python read_serial.py COM3 (Windows) or /dev/ttyUSB0 (Linux)")
+    #    sys.exit(1)
+
+    #port_name = sys.argv[1]
+    read_serial("/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00")

webui/app.py

@@ -0,0 +1,62 @@
+import subprocess
+from flask import Flask, Response
+
+app = Flask(__name__)
+
+def generate():
+    # FFmpeg command to capture the MJPEG stream without re-encoding.
+    command = [
+        'ffmpeg',
+        '-f', 'v4l2',
+        '-input_format', 'mjpeg', '-video_size', '1920x1080', '-framerate', '60.00',
+        '-i', '/dev/video0',
+        '-c', 'copy',
+        '-f', 'mjpeg',
+        'pipe:1'
+    ]
+    # Start the FFmpeg subprocess.
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=10**8)
+    data = b""
+    while True:
+        # Read raw bytes from FFmpeg's stdout.
+        chunk = process.stdout.read(1024)
+        if not chunk:
+            break
+        data += chunk
+
+        # Look for complete JPEG frames by finding start and end markers.
+        while True:
+            start = data.find(b'\xff\xd8')  # JPEG start
+            end = data.find(b'\xff\xd9')    # JPEG end
+            if start != -1 and end != -1 and end > start:
+                # Extract the JPEG frame.
+                jpg = data[start:end+2]
+                data = data[end+2:]
+                # Yield the frame with the required multipart MJPEG boundaries.
+                yield (b'--frame\r\n'
+                       b'Content-Type: image/jpeg\r\n\r\n' + jpg + b'\r\n')
+            else:
+                break
+
+@app.route('/video_feed')
+def video_feed():
+    # Set the MIME type to multipart so browsers render it as an MJPEG stream.
+    return Response(generate(), mimetype='multipart/x-mixed-replace; boundary=frame')
+
+@app.route('/')
+def index():
+    return """
+    <html>
+        <head>
+            <title>Webcam Stream</title>
+        </head>
+        <body>
+            <h1>Webcam Stream</h1>
+            <img src="/video_feed">
+        </body>
+    </html>
+    """
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)
+

webui/ipkvm/__init__.py

@@ -1,10 +1,12 @@
 from os import name, listdir
 from flask import Flask
+from flask_socketio import SocketIO
 import json
 import logging
 
-ui = Flask(__name__)
+app = Flask(__name__)
-logger = ui.logger
+ui = SocketIO(app)
+logger = app.logger
 logger.setLevel(logging.INFO)
 
 def new_profile():

webui/ipkvm/feed.py

@@ -21,8 +21,6 @@ class FrameBuffer(threading.Thread):
 
     def capture_feed(self):
         device = self.acquire_device()
-        print(device)
-        time.sleep(5)
         while True:
             # try:
             #     for frame in device.decode(video=0):
@@ -72,7 +70,7 @@ class FrameBuffer(threading.Thread):
         else:
             raise RuntimeError("We're on something other than Linux, and that's not yet supported!")
 
-        device.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
+        device.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"YUYV"))
         device.set(cv2.CAP_PROP_FRAME_WIDTH, int(profile["video_device"]["resolution"].split('x')[0]))
         device.set(cv2.CAP_PROP_FRAME_HEIGHT, int(profile["video_device"]["resolution"].split('x')[1]))
         device.set(cv2.CAP_PROP_FPS, float(profile["video_device"]["fps"]))

webui/ipkvm/routes.py

@@ -1,7 +1,10 @@
-from ipkvm import ui
+from ipkvm import app, ui
 from ipkvm import frame_buffer
-from flask import Response
+from flask import Response, render_template
 import time
+from ipkvm.util.mkb import HIDKeyCode
+import serial
+import json
 
 def generate_frames():
     while True:
@@ -9,22 +12,46 @@ def generate_frames():
         frame_buffer.new_frame.clear()
         yield (b'--frame\r\n'
                b'Content-Type: image/jpeg\r\n\r\n' + frame_buffer.cur_frame + b'\r\n')
+        
+@ui.on('key_down')
+def handle_keydown(data):
+    test_json_a = {
+      "mouseX": 99999,
+      "mouseY": 99999,
+      "mouse_down": ["rbutton", "lbutton"],
+      "mouse_up": ["otherbutton"],
+      "key_up": [],
+      "key_down": [HIDKeyCode[data]]
+    }
 
-@ui.route('/video_feed')
+    print(HIDKeyCode[data])
+    with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+        ser.write(json.dumps(test_json_a).encode())
+
+@ui.on('key_up')
+def handle_keyup(data):
+    test_json_a = {
+      "mouseX": 99999,
+      "mouseY": 99999,
+      "mouse_down": ["rbutton", "lbutton"],
+      "mouse_up": ["otherbutton"],
+      "key_up": [HIDKeyCode[data]],
+      "key_down": []
+    }
+
+    print(HIDKeyCode[data])
+    with serial.Serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00', 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+        ser.write(json.dumps(test_json_a).encode())
+
+@app.route('/video_feed')
 def video_feed():
     return Response(generate_frames(),
                     mimetype='multipart/x-mixed-replace; boundary=frame')
 
-@ui.route('/')
+@app.route('/')
 def index():
-    return """
+    return render_template('index.html')
-    <html>
+
-        <head>
+"""@socketio.on("connect")
-            <title>Webcam Stream</title>
+def kvm_client():
-        </head>
+    ui.start_background_task(mkb_handler)"""
-        <body>
-            <h1>Webcam Stream</h1>
-            <img src="/video_feed">
-        </body>
-    </html>
-    """

webui/ipkvm/static/mkb_handler.js

@@ -0,0 +1,30 @@
+const streamview = document.getElementById('streamview');
+
+var socket = io();
+
+function keydown_handler(event)
+{
+    console.log(`Key pressed: ${event.code}`);
+    socket.emit('key_down', event.code);
+}
+
+function keyup_handler(event)
+{
+    console.log(`Key released: ${event.code}`);
+    socket.emit('key_up', event.code);
+}
+
+function enable_listener()
+{
+    document.addEventListener("keydown", keydown_handler);
+    document.addEventListener("keyup", keyup_handler);
+}
+
+function disable_listener()
+{
+    document.removeEventListener("keydown", keydown_handler);
+    document.removeEventListener("keyup", keyup_handler);
+}
+
+streamview.addEventListener("mouseenter", enable_listener);
+streamview.addEventListener("mouseleave", disable_listener);

webui/ipkvm/static/style.css

@@ -0,0 +1,8 @@
+.stream-container {
+    margin: 0 auto;
+    padding: 20px;
+}
+.stream-view {
+    width: 100%;
+    height: auto;
+}

webui/ipkvm/templates/index.html

@@ -1,22 +1,14 @@
 <!DOCTYPE html>
+<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+<script src="{{ url_for('static', filename='vendor/socket.io.min.js') }}"></script>
 <html>
 <head>
-    <title>MJPEG Stream Viewer</title>
+    <title>IP KVM & OC Tuner</title>
-    <style>
-        .stream-container {
-            max-width: 800px;
-            margin: 0 auto;
-            padding: 20px;
-        }
-        .stream-view {
-            width: 100%;
-            height: auto;
-        }
-    </style>
 </head>
 <body>
-    <div class="stream-container">
+    <div id="streamview" class="stream-container">
-        <img src="{{ url_for('video_feed') }}" class="stream-view" />
+        <img src="/video_feed" class="stream-view" />
     </div>
 </body>
-</html>
+</html>
+<script src="{{ url_for('static', filename='mkb_handler.js') }}"></script>

webui/ipkvm/util/mkb.py

@@ -0,0 +1,134 @@
+from enum import IntEnum
+
+# God Bless CHADGPT
+class HIDKeyCode(IntEnum):
+    """
+    Enum that translates modern JS key.code values to HID scancodes.
+    """
+    # Letter keys (A-Z)
+    KeyA = 4
+    KeyB = 5
+    KeyC = 6
+    KeyD = 7
+    KeyE = 8
+    KeyF = 9
+    KeyG = 10
+    KeyH = 11
+    KeyI = 12
+    KeyJ = 13
+    KeyK = 14
+    KeyL = 15
+    KeyM = 16
+    KeyN = 17
+    KeyO = 18
+    KeyP = 19
+    KeyQ = 20
+    KeyR = 21
+    KeyS = 22
+    KeyT = 23
+    KeyU = 24
+    KeyV = 25
+    KeyW = 26
+    KeyX = 27
+    KeyY = 28
+    KeyZ = 29
+
+    # Number keys (top row)
+    Digit1 = 30
+    Digit2 = 31
+    Digit3 = 32
+    Digit4 = 33
+    Digit5 = 34
+    Digit6 = 35
+    Digit7 = 36
+    Digit8 = 37
+    Digit9 = 38
+    Digit0 = 39
+
+    # Control keys
+    Enter = 40
+    Escape = 41
+    Backspace = 42
+    Tab = 43
+    Space = 44
+
+    Minus = 45
+    Equal = 46
+    BracketLeft = 47
+    BracketRight = 48
+    Backslash = 49
+
+    # Punctuation keys
+    Semicolon = 51
+    Quote = 52
+    Backquote = 53
+    Comma = 54
+    Period = 55
+    Slash = 56
+
+    CapsLock = 57
+
+    # Function keys (F1-F12)
+    F1 = 58
+    F2 = 59
+    F3 = 60
+    F4 = 61
+    F5 = 62
+    F6 = 63
+    F7 = 64
+    F8 = 65
+    F9 = 66
+    F10 = 67
+    F11 = 68
+    F12 = 69
+
+    PrintScreen = 70
+    ScrollLock = 71
+    Pause = 72
+
+    Insert = 73
+    Home = 74
+    PageUp = 75
+
+    Delete = 76
+    End = 77
+    PageDown = 78
+
+    ArrowRight = 79
+    ArrowLeft = 80
+    ArrowDown = 81
+    ArrowUp = 82
+
+    # Numpad keys
+    NumLock = 83
+    NumpadDivide = 84
+    NumpadMultiply = 85
+    NumpadSubtract = 86
+    NumpadAdd = 87
+    NumpadEnter = 88
+    Numpad1 = 89
+    Numpad2 = 90
+    Numpad3 = 91
+    Numpad4 = 92
+    Numpad5 = 93
+    Numpad6 = 94
+    Numpad7 = 95
+    Numpad8 = 96
+    Numpad9 = 97
+    Numpad0 = 98
+    NumpadDecimal = 99
+
+    # Additional keys
+    IntlBackslash = 100
+    ContextMenu = 101
+    Power = 102
+
+    # Modifier keys
+    ControlLeft = 224
+    ShiftLeft = 225
+    AltLeft = 226
+    MetaLeft = 227  # Windows / Command key (left)
+    ControlRight = 228
+    ShiftRight = 229
+    AltRight = 230
+    MetaRight = 231  # Windows / Command key (right)

webui/ipkvm/util/super_serial.py

@@ -0,0 +1,2 @@
+import serial
+

webui/launch.py

@@ -1,4 +1,4 @@
-from ipkvm import ui
+from ipkvm import app, ui
 
 if __name__ == '__main__':
-    ui.run(host='0.0.0.0', port=5000)
+    ui.run(app, host='0.0.0.0', port=5000)

webui/serial_test.py

@@ -0,0 +1,52 @@
+import serial
+import sys
+import datetime
+import json
+import time
+
+test_json_a = {
+  "mouseX": 99999,
+  "mouseY": 99999,
+  "mouse_down": ["rbutton", "lbutton"],
+  "mouse_up": ["otherbutton"],
+  "key_up": [],
+  "key_down": [11, 12]
+}
+
+test_json_b = {
+  "mouseX": 99999,
+  "mouseY": 99999,
+  "mouse_down": ["rbutton", "lbutton"],
+  "mouse_up": ["otherbutton"],
+  "key_up": [11, 12],
+  "key_down": []
+}
+
+def read_serial(port):
+    try:
+        # Open the serial port
+        with serial.Serial(port, 115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE) as ser:
+            print(f"Listening on {port} at 115200 baud...")
+            while True:
+                # Read a line from the serial port
+                while ser.in_waiting > 0:
+                    line = ser.readline()   
+                    print(f'{datetime.datetime.now()} {line}')
+                # Print the raw data
+                ser.write(json.dumps(test_json_a).encode())
+                ser.write(json.dumps(test_json_b).encode())
+                time.sleep(1)
+    except serial.SerialException as e:
+        print(f"Error: {e}")
+    except KeyboardInterrupt:
+        print("Exiting...")
+        sys.exit()
+
+if __name__ == "__main__":
+    #if len(sys.argv) != 2:
+    #    print("Usage: python read_serial.py <port>")
+    #    print("Example: python read_serial.py COM3 (Windows) or /dev/ttyUSB0 (Linux)")
+    #    sys.exit(1)
+
+    #port_name = sys.argv[1]
+    read_serial('/dev/serial/by-id/usb-1a86_USB_Single_Serial_585D015807-if00')