Improve layer mask handling (#13065)

4 files changed, 38 insertions, 13 deletions

diff --git a/quantum/command.c b/quantum/command.c
index 34c4b36b1c..3a7dc0f8ca 100644
--- a/quantum/command.c
+++ b/quantum/command.c
@@ -781,6 +781,6 @@ uint8_t numkey2num(uint8_t code) {
 
 static void switch_default_layer(uint8_t layer) {
     xprintf("L%d\n", layer);
-    default_layer_set(1UL << layer);
+    default_layer_set((layer_state_t)1 << layer);
     clear_keyboard();
 }
diff --git a/quantum/quantum.c b/quantum/quantum.c
index 8ccdb774bd..b4cfa28d7d 100644
--- a/quantum/quantum.c
+++ b/quantum/quantum.c
@@ -340,13 +340,13 @@ void set_single_persistent_default_layer(uint8_t default_layer) {
 #if defined(AUDIO_ENABLE) && defined(DEFAULT_LAYER_SONGS)
     PLAY_SONG(default_layer_songs[default_layer]);
 #endif
-    eeconfig_update_default_layer(1U << default_layer);
-    default_layer_set(1U << default_layer);
+    eeconfig_update_default_layer((layer_state_t)1 << default_layer);
+    default_layer_set((layer_state_t)1 << default_layer);
 }
 
 layer_state_t update_tri_layer_state(layer_state_t state, uint8_t layer1, uint8_t layer2, uint8_t layer3) {
-    layer_state_t mask12 = (1UL << layer1) | (1UL << layer2);
-    layer_state_t mask3  = 1UL << layer3;
+    layer_state_t mask12 = ((layer_state_t)1 << layer1) | ((layer_state_t)1 << layer2);
+    layer_state_t mask3  = (layer_state_t)1 << layer3;
     return (state & mask12) == mask12 ? (state | mask3) : (state & ~mask3);
 }
 
diff --git a/tmk_core/common/action_layer.c b/tmk_core/common/action_layer.c
index af2d7d964b..ed1a4bd20d 100644
--- a/tmk_core/common/action_layer.c
+++ b/tmk_core/common/action_layer.c
@@ -131,32 +131,32 @@ bool layer_state_cmp(layer_state_t cmp_layer_state, uint8_t layer) {
     if (!cmp_layer_state) {
         return layer == 0;
     }
-    return (cmp_layer_state & (1UL << layer)) != 0;
+    return (cmp_layer_state & ((layer_state_t)1 << layer)) != 0;
 }
 
 /** \brief Layer move
  *
  * Turns on the given layer and turn off all other layers
  */
-void layer_move(uint8_t layer) { layer_state_set(1UL << layer); }
+void layer_move(uint8_t layer) { layer_state_set((layer_state_t)1 << layer); }
 
 /** \brief Layer on
  *
  * Turns on given layer
  */
-void layer_on(uint8_t layer) { layer_state_set(layer_state | (1UL << layer)); }
+void layer_on(uint8_t layer) { layer_state_set(layer_state | ((layer_state_t)1 << layer)); }
 
 /** \brief Layer off
  *
  * Turns off given layer
  */
-void layer_off(uint8_t layer) { layer_state_set(layer_state & ~(1UL << layer)); }
+void layer_off(uint8_t layer) { layer_state_set(layer_state & ~((layer_state_t)1 << layer)); }
 
 /** \brief Layer invert
  *
  * Toggle the given layer (set it if it's unset, or unset it if it's set)
  */
-void layer_invert(uint8_t layer) { layer_state_set(layer_state ^ (1UL << layer)); }
+void layer_invert(uint8_t layer) { layer_state_set(layer_state ^ ((layer_state_t)1 << layer)); }
 
 /** \brief Layer or
  *
@@ -258,7 +258,7 @@ uint8_t layer_switch_get_layer(keypos_t key) {
     layer_state_t layers = layer_state | default_layer_state;
     /* check top layer first */
     for (int8_t i = MAX_LAYER - 1; i >= 0; i--) {
-        if (layers & (1UL << i)) {
+        if (layers & ((layer_state_t)1 << i)) {
             action = action_for_key(i, key);
             if (action.code != ACTION_TRANSPARENT) {
                 return i;
diff --git a/tmk_core/common/action_layer.h b/tmk_core/common/action_layer.h
index d72cd3e3a5..b87d096eed 100644
--- a/tmk_core/common/action_layer.h
+++ b/tmk_core/common/action_layer.h
@@ -21,6 +21,29 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include "keyboard.h"
 #include "action.h"
 
+#ifdef DYNAMIC_KEYMAP_ENABLE
+#    ifndef DYNAMIC_KEYMAP_LAYER_COUNT
+#        define DYNAMIC_KEYMAP_LAYER_COUNT 4
+#    endif
+#    if DYNAMIC_KEYMAP_LAYER_COUNT <= 8
+#        ifndef LAYER_STATE_8BIT
+#            define LAYER_STATE_8BIT
+#        endif
+#    elif DYNAMIC_KEYMAP_LAYER_COUNT <= 16
+#        ifndef LAYER_STATE_16BIT
+#            define LAYER_STATE_16BIT
+#        endif
+#    else
+#        ifndef LAYER_STATE_32BIT
+#            define LAYER_STATE_32BIT
+#        endif
+#    endif
+#endif
+
+#if !defined(LAYER_STATE_8BIT) && !defined(LAYER_STATE_16BIT) && !defined(LAYER_STATE_32BIT)
+#    define LAYER_STATE_32BIT
+#endif
+
 #if defined(LAYER_STATE_8BIT)
 typedef uint8_t layer_state_t;
 #    define MAX_LAYER_BITS 3
@@ -35,13 +58,15 @@ typedef uint16_t layer_state_t;
 #        define MAX_LAYER 16
 #    endif
 #    define get_highest_layer(state) biton16(state)
-#else
+#elif defined(LAYER_STATE_32BIT)
 typedef uint32_t layer_state_t;
 #    define MAX_LAYER_BITS 5
 #    ifndef MAX_LAYER
 #        define MAX_LAYER 32
 #    endif
 #    define get_highest_layer(state) biton32(state)
+#else
+#    error Layer Mask size not specified.  HOW?!
 #endif
 
 /*
@@ -92,7 +117,7 @@ layer_state_t layer_state_set_kb(layer_state_t state);
 
 #    define layer_state_set(layer)
 #    define layer_state_is(layer) (layer == 0)
-#    define layer_state_cmp(state, layer) (state == 0 ? layer == 0 : (state & 1UL << layer) != 0)
+#    define layer_state_cmp(state, layer) (state == 0 ? layer == 0 : (state & (layer_state_t)1 << layer) != 0)
 
 #    define layer_debug()
 #    define layer_clear()