Extensible split data sync (#11930)

* Extensible split data sync capability through transactions.

- Split common transport has been split up between the transport layer
  and data layer.
- Split "transactions" model used, with convergence between I2C and
  serial data definitions.
- Slave matrix "generation count" is used to determine if the full slave
  matrix needs to be retrieved.
- Encoders get the same "generation count" treatment.
- All other blocks of data are synchronised when a change is detected.
- All transmissions have a globally-configurable deadline before a
  transmission is forced (`FORCED_SYNC_THROTTLE_MS`, default 100ms).
- Added atomicity for all core-synced data, preventing partial updates
- Added retries to AVR i2c_master's i2c_start, to minimise the number of
  failed transactions when interrupts are disabled on the slave due to
  atomicity checks.
- Some keyboards have had slight modifications made in order to ensure
  that they still build due to firmware size restrictions.

* Fixup LED_MATRIX compile.

* Parameterise ERROR_DISCONNECT_COUNT.

3 files changed, 38 insertions, 123 deletions

diff --git a/drivers/chibios/serial.c b/drivers/chibios/serial.c
index 54f7e1321f..f54fbcee4e 100644
--- a/drivers/chibios/serial.c
+++ b/drivers/chibios/serial.c
@@ -74,21 +74,12 @@ static THD_FUNCTION(Thread1, arg) {
     }
 }
 
-static SSTD_t *Transaction_table      = NULL;
-static uint8_t Transaction_table_size = 0;
-
-void soft_serial_initiator_init(SSTD_t *sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
-
+void soft_serial_initiator_init(void) {
     serial_output();
     serial_high();
 }
 
-void soft_serial_target_init(SSTD_t *sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
-
+void soft_serial_target_init(void) {
     serial_input();
 
     palEnablePadEvent(PAL_PORT(SOFT_SERIAL_PIN), PAL_PAD(SOFT_SERIAL_PIN), PAL_EVENT_MODE_FALLING_EDGE);
@@ -154,16 +145,14 @@ void interrupt_handler(void *arg) {
     uint8_t checksum_computed = 0;
     int     sstd_index        = 0;
 
-#ifdef SERIAL_USE_MULTI_TRANSACTION
     sstd_index = serial_read_byte();
     sync_send();
-#endif
 
-    SSTD_t *trans = &Transaction_table[sstd_index];
+    split_transaction_desc_t *trans = &split_transaction_table[sstd_index];
     for (int i = 0; i < trans->initiator2target_buffer_size; ++i) {
-        trans->initiator2target_buffer[i] = serial_read_byte();
+        split_trans_initiator2target_buffer(trans)[i] = serial_read_byte();
         sync_send();
-        checksum_computed += trans->initiator2target_buffer[i];
+        checksum_computed += split_trans_initiator2target_buffer(trans)[i];
     }
     checksum_computed ^= 7;
     uint8_t checksum_received = serial_read_byte();
@@ -172,12 +161,17 @@ void interrupt_handler(void *arg) {
     // wait for the sync to finish sending
     serial_delay();
 
+    // Allow any slave processing to occur
+    if (trans->slave_callback) {
+        trans->slave_callback(trans->initiator2target_buffer_size, split_trans_initiator2target_buffer(trans), trans->target2initiator_buffer_size, split_trans_target2initiator_buffer(trans));
+    }
+
     uint8_t checksum = 0;
     for (int i = 0; i < trans->target2initiator_buffer_size; ++i) {
-        serial_write_byte(trans->target2initiator_buffer[i]);
+        serial_write_byte(split_trans_target2initiator_buffer(trans)[i]);
         sync_send();
         serial_delay_half();
-        checksum += trans->target2initiator_buffer[i];
+        checksum += split_trans_target2initiator_buffer(trans)[i];
     }
     serial_write_byte(checksum ^ 7);
     sync_send();
@@ -206,15 +200,10 @@ void interrupt_handler(void *arg) {
 //    TRANSACTION_NO_RESPONSE
 //    TRANSACTION_DATA_ERROR
 // this code is very time dependent, so we need to disable interrupts
-#ifndef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_transaction(void) {
-    int sstd_index = 0;
-#else
 int soft_serial_transaction(int sstd_index) {
-#endif
-
-    if (sstd_index > Transaction_table_size) return TRANSACTION_TYPE_ERROR;
-    SSTD_t *trans = &Transaction_table[sstd_index];
+    if (sstd_index > NUM_TOTAL_TRANSACTIONS) return TRANSACTION_TYPE_ERROR;
+    split_transaction_desc_t *trans = &split_transaction_table[sstd_index];
+    if (!trans->status) return TRANSACTION_TYPE_ERROR;  // not registered
 
     // TODO: remove extra delay between transactions
     serial_delay();
@@ -244,14 +233,13 @@ int soft_serial_transaction(int sstd_index) {
 
     uint8_t checksum = 0;
     // send data to the slave
-#ifdef SERIAL_USE_MULTI_TRANSACTION
     serial_write_byte(sstd_index);  // first chunk is transaction id
     sync_recv();
-#endif
+
     for (int i = 0; i < trans->initiator2target_buffer_size; ++i) {
-        serial_write_byte(trans->initiator2target_buffer[i]);
+        serial_write_byte(split_trans_initiator2target_buffer(trans)[i]);
         sync_recv();
-        checksum += trans->initiator2target_buffer[i];
+        checksum += split_trans_initiator2target_buffer(trans)[i];
     }
     serial_write_byte(checksum ^ 7);
     sync_recv();
@@ -262,9 +250,9 @@ int soft_serial_transaction(int sstd_index) {
     // receive data from the slave
     uint8_t checksum_computed = 0;
     for (int i = 0; i < trans->target2initiator_buffer_size; ++i) {
-        trans->target2initiator_buffer[i] = serial_read_byte();
+        split_trans_target2initiator_buffer(trans)[i] = serial_read_byte();
         sync_recv();
-        checksum_computed += trans->target2initiator_buffer[i];
+        checksum_computed += split_trans_target2initiator_buffer(trans)[i];
     }
     checksum_computed ^= 7;
     uint8_t checksum_received = serial_read_byte();
diff --git a/drivers/chibios/serial.h b/drivers/chibios/serial.h
deleted file mode 100644
index 0c1857d52e..0000000000
--- a/drivers/chibios/serial.h
+++ /dev/null
@@ -1,62 +0,0 @@
-#pragma once
-
-#include <stdbool.h>
-
-// /////////////////////////////////////////////////////////////////
-// Need Soft Serial defines in config.h
-// /////////////////////////////////////////////////////////////////
-// ex.
-//  #define SOFT_SERIAL_PIN ??   // ?? = D0,D1,D2,D3,E6
-//  OPTIONAL: #define SELECT_SOFT_SERIAL_SPEED ? // ? = 1,2,3,4,5
-//                                               //  1: about 137kbps (default)
-//                                               //  2: about 75kbps
-//                                               //  3: about 39kbps
-//                                               //  4: about 26kbps
-//                                               //  5: about 20kbps
-//
-// //// USE simple API (using signle-type transaction function)
-//   /* nothing */
-// //// USE flexible API (using multi-type transaction function)
-//   #define SERIAL_USE_MULTI_TRANSACTION
-//
-// /////////////////////////////////////////////////////////////////
-
-// Soft Serial Transaction Descriptor
-typedef struct _SSTD_t {
-    uint8_t *status;
-    uint8_t  initiator2target_buffer_size;
-    uint8_t *initiator2target_buffer;
-    uint8_t  target2initiator_buffer_size;
-    uint8_t *target2initiator_buffer;
-} SSTD_t;
-#define TID_LIMIT(table) (sizeof(table) / sizeof(SSTD_t))
-
-// initiator is transaction start side
-void soft_serial_initiator_init(SSTD_t *sstd_table, int sstd_table_size);
-// target is interrupt accept side
-void soft_serial_target_init(SSTD_t *sstd_table, int sstd_table_size);
-
-// initiator result
-#define TRANSACTION_END 0
-#define TRANSACTION_NO_RESPONSE 0x1
-#define TRANSACTION_DATA_ERROR 0x2
-#define TRANSACTION_TYPE_ERROR 0x4
-#ifndef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_transaction(void);
-#else
-int soft_serial_transaction(int sstd_index);
-#endif
-
-// target status
-// *SSTD_t.status has
-//   initiator:
-//       TRANSACTION_END
-//    or TRANSACTION_NO_RESPONSE
-//    or TRANSACTION_DATA_ERROR
-//   target:
-//       TRANSACTION_DATA_ERROR
-//    or TRANSACTION_ACCEPTED
-#define TRANSACTION_ACCEPTED 0x8
-#ifdef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_get_and_clean_status(int sstd_index);
-#endif
diff --git a/drivers/chibios/serial_usart.c b/drivers/chibios/serial_usart.c
index cae29388c3..9f180d2d74 100644
--- a/drivers/chibios/serial_usart.c
+++ b/drivers/chibios/serial_usart.c
@@ -113,37 +113,29 @@ void usart_slave_init(void) {
     chThdCreateStatic(waSlaveThread, sizeof(waSlaveThread), HIGHPRIO, SlaveThread, NULL);
 }
 
-static SSTD_t* Transaction_table      = NULL;
-static uint8_t Transaction_table_size = 0;
+void soft_serial_initiator_init(void) { usart_master_init(); }
 
-void soft_serial_initiator_init(SSTD_t* sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
-
-    usart_master_init();
-}
-
-void soft_serial_target_init(SSTD_t* sstd_table, int sstd_table_size) {
-    Transaction_table      = sstd_table;
-    Transaction_table_size = (uint8_t)sstd_table_size;
-
-    usart_slave_init();
-}
+void soft_serial_target_init(void) { usart_slave_init(); }
 
 void handle_soft_serial_slave(void) {
-    uint8_t sstd_index = sdGet(&SERIAL_USART_DRIVER);  // first chunk is always transaction id
-    SSTD_t* trans      = &Transaction_table[sstd_index];
+    uint8_t                   sstd_index = sdGet(&SERIAL_USART_DRIVER);  // first chunk is always transaction id
+    split_transaction_desc_t* trans      = &split_transaction_table[sstd_index];
 
     // Always write back the sstd_index as part of a basic handshake
     sstd_index ^= HANDSHAKE_MAGIC;
     sdWrite(&SERIAL_USART_DRIVER, &sstd_index, sizeof(sstd_index));
 
     if (trans->initiator2target_buffer_size) {
-        sdRead(&SERIAL_USART_DRIVER, trans->initiator2target_buffer, trans->initiator2target_buffer_size);
+        sdRead(&SERIAL_USART_DRIVER, split_trans_initiator2target_buffer(trans), trans->initiator2target_buffer_size);
+    }
+
+    // Allow any slave processing to occur
+    if (trans->slave_callback) {
+        trans->slave_callback(trans->initiator2target_buffer_size, split_trans_initiator2target_buffer(trans), trans->target2initiator_buffer_size, split_trans_target2initiator_buffer(trans));
     }
 
     if (trans->target2initiator_buffer_size) {
-        sdWrite(&SERIAL_USART_DRIVER, trans->target2initiator_buffer, trans->target2initiator_buffer_size);
+        sdWrite(&SERIAL_USART_DRIVER, split_trans_target2initiator_buffer(trans), trans->target2initiator_buffer_size);
     }
 
     if (trans->status) {
@@ -160,17 +152,14 @@ void handle_soft_serial_slave(void) {
 //    TRANSACTION_END
 //    TRANSACTION_NO_RESPONSE
 //    TRANSACTION_DATA_ERROR
-#ifndef SERIAL_USE_MULTI_TRANSACTION
-int soft_serial_transaction(void) {
-    uint8_t sstd_index = 0;
-#else
 int soft_serial_transaction(int index) {
     uint8_t sstd_index = index;
-#endif
 
-    if (sstd_index > Transaction_table_size) return TRANSACTION_TYPE_ERROR;
-    SSTD_t* trans = &Transaction_table[sstd_index];
-    msg_t   res   = 0;
+    if (sstd_index > NUM_TOTAL_TRANSACTIONS) return TRANSACTION_TYPE_ERROR;
+    split_transaction_desc_t* trans = &split_transaction_table[sstd_index];
+    msg_t                     res   = 0;
+
+    if (!trans->status) return TRANSACTION_TYPE_ERROR;  // not registered
 
     sdClear(&SERIAL_USART_DRIVER);
 
@@ -189,7 +178,7 @@ int soft_serial_transaction(int index) {
     }
 
     if (trans->initiator2target_buffer_size) {
-        res = sdWriteTimeout(&SERIAL_USART_DRIVER, trans->initiator2target_buffer, trans->initiator2target_buffer_size, TIME_MS2I(SERIAL_USART_TIMEOUT));
+        res = sdWriteTimeout(&SERIAL_USART_DRIVER, split_trans_initiator2target_buffer(trans), trans->initiator2target_buffer_size, TIME_MS2I(SERIAL_USART_TIMEOUT));
         if (res < 0) {
             dprintf("serial::usart_transmit NO_RESPONSE\n");
             return TRANSACTION_NO_RESPONSE;
@@ -197,7 +186,7 @@ int soft_serial_transaction(int index) {
     }
 
     if (trans->target2initiator_buffer_size) {
-        res = sdReadTimeout(&SERIAL_USART_DRIVER, trans->target2initiator_buffer, trans->target2initiator_buffer_size, TIME_MS2I(SERIAL_USART_TIMEOUT));
+        res = sdReadTimeout(&SERIAL_USART_DRIVER, split_trans_target2initiator_buffer(trans), trans->target2initiator_buffer_size, TIME_MS2I(SERIAL_USART_TIMEOUT));
         if (res < 0) {
             dprintf("serial::usart_receive NO_RESPONSE\n");
             return TRANSACTION_NO_RESPONSE;