Brute force update CLI tools

16 files changed, 374 insertions, 124 deletions

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 1757792b31..77724a2448 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -14,6 +14,7 @@ from . import config
 from . import docs
 from . import doctor
 from . import flash
+from . import generate
 from . import hello
 from . import info
 from . import json
diff --git a/lib/python/qmk/cli/c2json.py b/lib/python/qmk/cli/c2json.py
index 0267303fd2..8c8bd1f57e 100644
--- a/lib/python/qmk/cli/c2json.py
+++ b/lib/python/qmk/cli/c2json.py
@@ -44,7 +44,7 @@ def c2json(cli):
 
     # Generate the keymap.json
     try:
-        keymap_json = qmk.keymap.generate(keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'], type='json', keymap=keymap_json['keymap'])
+        keymap_json = qmk.keymap.generate_json(keymap_json['keymap'], keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'])
     except KeyError:
         cli.log.error('Something went wrong. Try to use --no-cpp.')
         sys.exit(1)
diff --git a/lib/python/qmk/cli/generate/__init__.py b/lib/python/qmk/cli/generate/__init__.py
new file mode 100644
index 0000000000..13bd1f0914
--- /dev/null
+++ b/lib/python/qmk/cli/generate/__init__.py
@@ -0,0 +1,2 @@
+from . import api
+from . import docs
diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py
new file mode 100755
index 0000000000..9807a9cd68
--- /dev/null
+++ b/lib/python/qmk/cli/generate/api.py
@@ -0,0 +1,58 @@
+"""This script automates the generation of the QMK API data.
+"""
+from pathlib import Path
+from shutil import copyfile
+import json
+
+from milc import cli
+
+from qmk.datetime import current_datetime
+from qmk.info import info_json
+from qmk.keyboard import list_keyboards
+
+
+@cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True)
+def generate_api(cli):
+    """Generates the QMK API data.
+    """
+    api_data_dir = Path('api_data')
+    v1_dir = api_data_dir / 'v1'
+    keyboard_list = v1_dir / 'keyboard_list.json'
+    keyboard_all = v1_dir / 'keyboards.json'
+    usb_file = v1_dir / 'usb.json'
+
+    if not api_data_dir.exists():
+        api_data_dir.mkdir()
+
+    kb_all = {'last_updated': current_datetime(), 'keyboards': {}}
+    usb_list = {'last_updated': current_datetime(), 'devices': {}}
+
+    # Generate and write keyboard specific JSON files
+    for keyboard_name in list_keyboards():
+        kb_all['keyboards'][keyboard_name] = info_json(keyboard_name)
+        keyboard_dir = v1_dir / 'keyboards' / keyboard_name
+        keyboard_info = keyboard_dir / 'info.json'
+        keyboard_readme = keyboard_dir / 'readme.md'
+        keyboard_readme_src = Path('keyboards') / keyboard_name / 'readme.md'
+
+        keyboard_dir.mkdir(parents=True, exist_ok=True)
+        keyboard_info.write_text(json.dumps(kb_all['keyboards'][keyboard_name]))
+
+        if keyboard_readme_src.exists():
+            copyfile(keyboard_readme_src, keyboard_readme)
+
+        if 'usb' in kb_all['keyboards'][keyboard_name]:
+            usb = kb_all['keyboards'][keyboard_name]['usb']
+
+            if usb['vid'] not in usb_list['devices']:
+                usb_list['devices'][usb['vid']] = {}
+
+            if usb['pid'] not in usb_list['devices'][usb['vid']]:
+                usb_list['devices'][usb['vid']][usb['pid']] = {}
+
+            usb_list['devices'][usb['vid']][usb['pid']][keyboard_name] = usb
+
+    # Write the global JSON files
+    keyboard_list.write_text(json.dumps({'last_updated': current_datetime(), 'keyboards': sorted(kb_all['keyboards'])}))
+    keyboard_all.write_text(json.dumps(kb_all))
+    usb_file.write_text(json.dumps(usb_list))
diff --git a/lib/python/qmk/cli/generate/docs.py b/lib/python/qmk/cli/generate/docs.py
new file mode 100644
index 0000000000..a59a24db50
--- /dev/null
+++ b/lib/python/qmk/cli/generate/docs.py
@@ -0,0 +1,37 @@
+"""Build QMK documentation locally
+"""
+import shutil
+import subprocess
+from pathlib import Path
+
+from milc import cli
+
+DOCS_PATH = Path('docs/')
+BUILD_PATH = Path('.build/docs/')
+
+
+@cli.subcommand('Build QMK documentation.', hidden=False if cli.config.user.developer else True)
+def generate_docs(cli):
+    """Invoke the docs generation process
+
+    TODO(unclaimed):
+        * [ ] Add a real build step... something static docs
+    """
+
+    if BUILD_PATH.exists():
+        shutil.rmtree(BUILD_PATH)
+
+    shutil.copytree(DOCS_PATH, BUILD_PATH)
+
+    # When not verbose we want to hide all output
+    args = {'check': True}
+    if not cli.args.verbose:
+        args.update({'stdout': subprocess.DEVNULL, 'stderr': subprocess.STDOUT})
+
+    cli.log.info('Generating internal docs...')
+
+    # Generate internal docs
+    subprocess.run(['doxygen', 'Doxyfile'], **args)
+    subprocess.run(['moxygen', '-q', '-a', '-g', '-o', BUILD_PATH / 'internals_%s.md', 'doxygen/xml'], **args)
+
+    cli.log.info('Successfully generated internal docs to %s.', BUILD_PATH)
diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py
index 92f0b25c0f..9ab299a21e 100755
--- a/lib/python/qmk/cli/info.py
+++ b/lib/python/qmk/cli/info.py
@@ -19,7 +19,7 @@ ROW_LETTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop'
 COL_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijilmnopqrstuvwxyz'
 
 
-def show_keymap(info_json, title_caps=True):
+def show_keymap(kb_info_json, title_caps=True):
     """Render the keymap in ascii art.
     """
     keymap_path = locate_keymap(cli.config.info.keyboard, cli.config.info.keymap)
@@ -51,10 +51,10 @@ def show_layouts(kb_info_json, title_caps=True):
         print(layout_art)  # Avoid passing dirty data to cli.echo()
 
 
-def show_matrix(info_json, title_caps=True):
+def show_matrix(kb_info_json, title_caps=True):
     """Render the layout with matrix labels in ascii art.
     """
-    for layout_name, layout in info_json['layouts'].items():
+    for layout_name, layout in kb_info_json['layouts'].items():
         # Build our label list
         labels = []
         for key in layout['layout']:
@@ -75,51 +75,51 @@ def show_matrix(info_json, title_caps=True):
         print(render_layout(kb_info_json['layouts'][layout_name]['layout'], cli.config.info.ascii, labels))
 
 
-def print_friendly_output(info_json):
+def print_friendly_output(kb_info_json):
     """Print the info.json in a friendly text format.
     """
-    cli.echo('{fg_blue}Keyboard Name{fg_reset}: %s', info_json.get('keyboard_name', 'Unknown'))
-    cli.echo('{fg_blue}Manufacturer{fg_reset}: %s', info_json.get('manufacturer', 'Unknown'))
-    if 'url' in info_json:
-        cli.echo('{fg_blue}Website{fg_reset}: %s', info_json.get('url', ''))
-    if info_json.get('maintainer', 'qmk') == 'qmk':
+    cli.echo('{fg_blue}Keyboard Name{fg_reset}: %s', kb_info_json.get('keyboard_name', 'Unknown'))
+    cli.echo('{fg_blue}Manufacturer{fg_reset}: %s', kb_info_json.get('manufacturer', 'Unknown'))
+    if 'url' in kb_info_json:
+        cli.echo('{fg_blue}Website{fg_reset}: %s', kb_info_json.get('url', ''))
+    if kb_info_json.get('maintainer', 'qmk') == 'qmk':
         cli.echo('{fg_blue}Maintainer{fg_reset}: QMK Community')
     else:
-        cli.echo('{fg_blue}Maintainer{fg_reset}: %s', info_json['maintainer'])
-    cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', info_json.get('keyboard_folder', 'Unknown'))
-    cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(info_json['layouts'].keys())))
-    if 'width' in info_json and 'height' in info_json:
-        cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (info_json['width'], info_json['height']))
-    cli.echo('{fg_blue}Processor{fg_reset}: %s', info_json.get('processor', 'Unknown'))
-    cli.echo('{fg_blue}Bootloader{fg_reset}: %s', info_json.get('bootloader', 'Unknown'))
+        cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer'])
+    cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown'))
+    cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
+    if 'width' in kb_info_json and 'height' in kb_info_json:
+        cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height']))
+    cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown'))
+    cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown'))
 
     if cli.config.info.layouts:
-        show_layouts(info_json, True)
+        show_layouts(kb_info_json, True)
 
     if cli.config.info.matrix:
-        show_matrix(info_json, True)
+        show_matrix(kb_info_json, True)
 
     if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file':
-        show_keymap(info_json, True)
+        show_keymap(kb_info_json, True)
 
 
-def print_text_output(info_json):
+def print_text_output(kb_info_json):
     """Print the info.json in a plain text format.
     """
-    for key in sorted(info_json):
+    for key in sorted(kb_info_json):
         if key == 'layouts':
-            cli.echo('{fg_blue}layouts{fg_reset}: %s', ', '.join(sorted(info_json['layouts'].keys())))
+            cli.echo('{fg_blue}layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
         else:
-            cli.echo('{fg_blue}%s{fg_reset}: %s', key, info_json[key])
+            cli.echo('{fg_blue}%s{fg_reset}: %s', key, kb_info_json[key])
 
     if cli.config.info.layouts:
-        show_layouts(info_json, False)
+        show_layouts(kb_info_json, False)
 
     if cli.config.info.matrix:
-        show_matrix(info_json, False)
+        show_matrix(kb_info_json, False)
 
     if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file':
-        show_keymap(info_json, False)
+        show_keymap(kb_info_json, False)
 
 
 @cli.argument('-kb', '--keyboard', help='Keyboard to show info for.')
diff --git a/lib/python/qmk/cli/json2c.py b/lib/python/qmk/cli/json2c.py
index 2a90094368..426078063c 100755
--- a/lib/python/qmk/cli/json2c.py
+++ b/lib/python/qmk/cli/json2c.py
@@ -38,7 +38,7 @@ def json2c(cli):
         user_keymap = json.load(fd)
 
     # Generate the keymap
-    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+    keymap_c = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
 
     if cli.args.output:
         cli.args.output.parent.mkdir(parents=True, exist_ok=True)
diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py
index ca0c5661a4..8b6c451673 100644
--- a/lib/python/qmk/cli/list/keyboards.py
+++ b/lib/python/qmk/cli/list/keyboards.py
@@ -1,28 +1,13 @@
 """List the keyboards currently defined within QMK
 """
-# We avoid pathlib here because this is performance critical code.
-import os
-import glob
-
 from milc import cli
 
-BASE_PATH = os.path.join(os.getcwd(), "keyboards") + os.path.sep
-KB_WILDCARD = os.path.join(BASE_PATH, "**", "rules.mk")
-
-
-def find_name(path):
-    """Determine the keyboard name by stripping off the base_path and rules.mk.
-    """
-    return path.replace(BASE_PATH, "").replace(os.path.sep + "rules.mk", "")
+import qmk.keyboard
 
 
 @cli.subcommand("List the keyboards currently defined within QMK")
 def list_keyboards(cli):
     """List the keyboards currently defined within QMK
     """
-    # find everywhere we have rules.mk where keymaps isn't in the path
-    paths = [path for path in glob.iglob(KB_WILDCARD, recursive=True) if 'keymaps' not in path]
-
-    # Extract the keyboard name from the path and print it
-    for keyboard_name in sorted(map(find_name, paths)):
+    for keyboard_name in qmk.keyboard.list_keyboards():
         print(keyboard_name)
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 102111d7c4..94ab68e5e4 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -12,3 +12,8 @@ MAX_KEYBOARD_SUBFOLDERS = 5
 CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F411'
 LUFA_PROCESSORS = 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None
 VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
+
+# Common format strings
+DATE_FORMAT = '%Y-%m-%d'
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
+TIME_FORMAT = '%H:%M:%S'
diff --git a/lib/python/qmk/datetime.py b/lib/python/qmk/datetime.py
new file mode 100644
index 0000000000..4bffcc6217
--- /dev/null
+++ b/lib/python/qmk/datetime.py
@@ -0,0 +1,29 @@
+"""Functions to work with dates and times in a uniform way.
+
+The results of these functions are cached for 5 seconds to provide uniform time strings across short running processes. Long running processes that need more precise timekeeping should not use these functions.
+"""
+from time import gmtime, strftime
+
+from qmk.constants import DATE_FORMAT, DATETIME_FORMAT, TIME_FORMAT
+from qmk.decorators import lru_cache
+
+
+@lru_cache(timeout=5)
+def current_date():
+    """Returns the current time in UTZ as a formatted string.
+    """
+    return strftime(DATE_FORMAT, gmtime())
+
+
+@lru_cache(timeout=5)
+def current_datetime():
+    """Returns the current time in UTZ as a formatted string.
+    """
+    return strftime(DATETIME_FORMAT, gmtime())
+
+
+@lru_cache(timeout=5)
+def current_time():
+    """Returns the current time in UTZ as a formatted string.
+    """
+    return strftime(TIME_FORMAT, gmtime())
diff --git a/lib/python/qmk/decorators.py b/lib/python/qmk/decorators.py
index f8f2facb1c..629402b095 100644
--- a/lib/python/qmk/decorators.py
+++ b/lib/python/qmk/decorators.py
@@ -2,6 +2,7 @@
 """
 import functools
 from pathlib import Path
+from time import monotonic
 
 from milc import cli
 
@@ -84,3 +85,38 @@ def automagic_keymap(func):
         return func(*args, **kwargs)
 
     return wrapper
+
+
+def lru_cache(timeout=10, maxsize=128, typed=False):
+    """Least Recently Used Cache- cache the result of a function.
+
+    Args:
+
+        timeout
+            How many seconds to cache results for.
+
+        maxsize
+            The maximum size of the cache in bytes
+
+        typed
+            When `True` argument types will be taken into consideration, for example `3` and `3.0` will be treated as different keys.
+    """
+    def wrapper_cache(func):
+        func = functools.lru_cache(maxsize=maxsize, typed=typed)(func)
+        func.expiration = monotonic() + timeout
+
+        @functools.wraps(func)
+        def wrapped_func(*args, **kwargs):
+            if monotonic() >= func.expiration:
+                func.expiration = monotonic() + timeout
+
+                func.cache_clear()
+
+            return func(*args, **kwargs)
+
+        wrapped_func.cache_info = func.cache_info
+        wrapped_func.cache_clear = func.cache_clear
+
+        return wrapped_func
+
+    return wrapper_cache
diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index e8a44a33f9..d73ba8cfb6 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -9,6 +9,7 @@ from milc import cli
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts
 from qmk.keyboard import config_h, rules_mk
+from qmk.keymap import list_keymaps
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
 
@@ -25,6 +26,7 @@ def info_json(keyboard):
     info_data = {
         'keyboard_name': str(keyboard),
         'keyboard_folder': str(keyboard),
+        'keymaps': {},
         'layouts': {},
         'parse_errors': [],
         'parse_warnings': [],
@@ -40,6 +42,7 @@ def info_json(keyboard):
         if not layout_name.startswith('LAYOUT_kc'):
             info_data['layouts'][layout_name] = layout_json
 
+    # Merge in the data from info.json, config.h, and rules.mk
     info_data = merge_info_jsons(keyboard, info_data)
     info_data = _extract_config_h(info_data)
     info_data = _extract_rules_mk(info_data)
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 33f182e8f9..a4c2873757 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -3,6 +3,8 @@
 from array import array
 from math import ceil
 from pathlib import Path
+import os
+from glob import glob
 
 from qmk.c_parse import parse_config_h_file
 from qmk.makefile import parse_rules_mk_file
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 166697ee6a..31c61ae6a8 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -29,33 +29,37 @@ __KEYMAP_GOES_HERE__
 """
 
 
-def template(keyboard, type='c'):
-    """Returns the `keymap.c` or `keymap.json` template for a keyboard.
+def template_json(keyboard):
+    """Returns a `keymap.json` template for a keyboard.
 
-    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
-    text will be used instead of `DEFAULT_KEYMAP_C`.
-
-    If a template exists in `keyboards/<keyboard>/templates/keymap.json` that
-    text will be used instead of an empty dictionary.
+    If a template exists in `keyboards/<keyboard>/templates/keymap.json` that text will be used instead of an empty dictionary.
 
     Args:
         keyboard
             The keyboard to return a template for.
+    """
+    template_file = Path('keyboards/%s/templates/keymap.json' % keyboard)
+    template = {'keyboard': keyboard}
+    if template_file.exists():
+        template.update(json.loads(template_file.read_text()))
+
+    return template
+
 
-        type
-            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
+def template_c(keyboard):
+    """Returns a `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that text will be used instead of an empty dictionary.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
     """
-    if type == 'json':
-        template_file = Path('keyboards/%s/templates/keymap.json' % keyboard)
-        template = {'keyboard': keyboard}
-        if template_file.exists():
-            template.update(json.loads(template_file.read_text()))
+    template_file = Path('keyboards/%s/templates/keymap.c' % keyboard)
+    if template_file.exists():
+        template = template_file.read_text()
     else:
-        template_file = Path('keyboards/%s/templates/keymap.c' % keyboard)
-        if template_file.exists():
-            template = template_file.read_text()
-        else:
-            template = DEFAULT_KEYMAP_C
+        template = DEFAULT_KEYMAP_C
 
     return template
 
@@ -69,15 +73,65 @@ def _strip_any(keycode):
     return keycode
 
 
-def is_keymap_dir(keymap):
+def is_keymap_dir(keymap, c=True, json=True, additional_files=None):
     """Return True if Path object `keymap` has a keymap file inside.
+
+    Args:
+        keymap
+            A Path() object for the keymap directory you want to check.
+
+        c
+            When true include `keymap.c` keymaps.
+
+        json
+            When true include `keymap.json` keymaps.
+
+        additional_files
+            A sequence of additional filenames to check against to determine if a directory is a keymap. All files must exist for a match to happen. For example, if you want to match a C keymap with both a `config.h` and `rules.mk` file: `is_keymap_dir(keymap_dir, json=False, additional_files=['config.h', 'rules.mk'])`
     """
-    for file in ('keymap.c', 'keymap.json'):
+    files = []
+
+    if c:
+        files.append('keymap.c')
+
+    if json:
+        files.append('keymap.json')
+
+    for file in files:
         if (keymap / file).is_file():
+            if additional_files:
+                for file in additional_files:
+                    if not (keymap / file).is_file():
+                        return False
+
             return True
 
 
-def generate(keyboard, layout, layers, type='c', keymap=None):
+def generate_json(keymap, keyboard, layout, layers):
+    """Returns a `keymap.json` for the specified keyboard, layout, and layers.
+
+    Args:
+        keymap
+            A name for this keymap.
+
+        keyboard
+            The name of the keyboard.
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    new_keymap = template_json(keyboard)
+    new_keymap['keymap'] = keymap
+    new_keymap['layout'] = layout
+    new_keymap['layers'] = layers
+
+    return new_keymap
+
+
+def generate_c(keyboard, layout, layers):
     """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
 
     Args:
@@ -89,33 +143,33 @@ def generate(keyboard, layout, layers, type='c', keymap=None):
 
         layers
             An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
-
-        type
-            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
     """
-    new_keymap = template(keyboard, type)
-    if type == 'json':
-        new_keymap['keymap'] = keymap
-        new_keymap['layout'] = layout
-        new_keymap['layers'] = layers
-    else:
-        layer_txt = []
-        for layer_num, layer in enumerate(layers):
-            if layer_num != 0:
-                layer_txt[-1] = layer_txt[-1] + ','
+    new_keymap = template_c(keyboard)
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer = map(_strip_any, layer)
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)
 
-            layer = map(_strip_any, layer)
-            layer_keys = ', '.join(layer)
-            layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+    return new_keymap
 
-        keymap = '\n'.join(layer_txt)
-        new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)
 
-    return new_keymap
+def write_file(keymap_filename, keymap_content):
+    keymap_filename.parent.mkdir(parents=True, exist_ok=True)
+    keymap_filename.write_text(keymap_content)
 
+    cli.log.info('Wrote keymap to {fg_cyan}%s', keymap_filename)
+
+    return keymap_filename
 
-def write(keyboard, keymap, layout, layers, type='c'):
-    """Generate the `keymap.c` and write it to disk.
+
+def write_json(keyboard, keymap, layout, layers):
+    """Generate the `keymap.json` and write it to disk.
 
     Returns the filename written to.
 
@@ -131,23 +185,36 @@ def write(keyboard, keymap, layout, layers, type='c'):
 
         layers
             An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
-
-        type
-            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
     """
-    keymap_content = generate(keyboard, layout, layers, type)
-    if type == 'json':
-        keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'
-        keymap_content = json.dumps(keymap_content)
-    else:
-        keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
+    keymap_json = generate_json(keyboard, keymap, layout, layers)
+    keymap_content = json.dumps(keymap_json)
+    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'
+
+    return write_file(keymap_file, keymap_content)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
 
-    keymap_file.parent.mkdir(parents=True, exist_ok=True)
-    keymap_file.write_text(keymap_content)
+        keymap
+            The name of the keymap
 
-    cli.log.info('Wrote keymap to {fg_cyan}%s', keymap_file)
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_content = generate_c(keyboard, layout, layers)
+    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
 
-    return keymap_file
+    return write_file(keymap_file, keymap_content)
 
 
 def locate_keymap(keyboard, keymap):
@@ -189,38 +256,58 @@ def locate_keymap(keyboard, keymap):
                     return community_layout / 'keymap.c'
 
 
-def list_keymaps(keyboard):
-    """ List the available keymaps for a keyboard.
+def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
+    """List the available keymaps for a keyboard.
 
     Args:
-        keyboard: the keyboards full name with vendor and revision if necessary, example: clueboard/66/rev3
+        keyboard
+            The keyboards full name with vendor and revision if necessary, example: clueboard/66/rev3
+
+        c
+            When true include `keymap.c` keymaps.
+
+        json
+            When true include `keymap.json` keymaps.
+
+        additional_files
+            A sequence of additional filenames to check against to determine if a directory is a keymap. All files must exist for a match to happen. For example, if you want to match a C keymap with both a `config.h` and `rules.mk` file: `is_keymap_dir(keymap_dir, json=False, additional_files=['config.h', 'rules.mk'])`
+
+        fullpath
+            When set to True the full path of the keymap relative to the `qmk_firmware` root will be provided.
 
     Returns:
-        a set with the names of the available keymaps
+        a sorted list of valid keymap names.
     """
     # parse all the rules.mk files for the keyboard
     rules = rules_mk(keyboard)
     names = set()
 
     if rules:
-        # qmk_firmware/keyboards
         keyboards_dir = Path('keyboards')
-        # path to the keyboard's directory
         kb_path = keyboards_dir / keyboard
+
         # walk up the directory tree until keyboards_dir
         # and collect all directories' name with keymap.c file in it
         while kb_path != keyboards_dir:
             keymaps_dir = kb_path / "keymaps"
-            if keymaps_dir.exists():
-                names = names.union([keymap.name for keymap in keymaps_dir.iterdir() if is_keymap_dir(keymap)])
+
+            if keymaps_dir.is_dir():
+                for keymap in keymaps_dir.iterdir():
+                    if is_keymap_dir(keymap, c, json, additional_files):
+                        keymap = keymap if fullpath else keymap.name
+                        names.add(keymap)
+
             kb_path = kb_path.parent
 
         # if community layouts are supported, get them
         if "LAYOUTS" in rules:
             for layout in rules["LAYOUTS"].split():
                 cl_path = Path('layouts/community') / layout
-                if cl_path.exists():
-                    names = names.union([keymap.name for keymap in cl_path.iterdir() if is_keymap_dir(keymap)])
+                if cl_path.is_dir():
+                    for keymap in cl_path.iterdir():
+                        if is_keymap_dir(keymap, c, json, additional_files):
+                            keymap = keymap if fullpath else keymap.name
+                            names.add(keymap)
 
     return sorted(names)
 
diff --git a/lib/python/qmk/questions.py b/lib/python/qmk/questions.py
index 27f43ac1e9..865c6bbdc5 100644
--- a/lib/python/qmk/questions.py
+++ b/lib/python/qmk/questions.py
@@ -1,7 +1,12 @@
 """Functions to collect user input.
 """
 
-from milc import cli, format_ansi
+from milc import cli
+
+try:
+    from milc import format_ansi
+except ImportError:
+    from milc.ansi import format_ansi
 
 
 def yesno(prompt, *args, default=None, **kwargs):
diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py
index 7ef708e0de..f1ecf29378 100644
--- a/lib/python/qmk/tests/test_qmk_keymap.py
+++ b/lib/python/qmk/tests/test_qmk_keymap.py
@@ -1,33 +1,33 @@
 import qmk.keymap
 
 
-def test_template_onekey_proton_c():
-    templ = qmk.keymap.template('handwired/onekey/proton_c')
+def test_template_c_onekey_proton_c():
+    templ = qmk.keymap.template_c('handwired/onekey/proton_c')
     assert templ == qmk.keymap.DEFAULT_KEYMAP_C
 
 
-def test_template_onekey_proton_c_json():
-    templ = qmk.keymap.template('handwired/onekey/proton_c', type='json')
+def test_template_json_onekey_proton_c():
+    templ = qmk.keymap.template_json('handwired/onekey/proton_c')
     assert templ == {'keyboard': 'handwired/onekey/proton_c'}
 
 
-def test_template_onekey_pytest():
-    templ = qmk.keymap.template('handwired/onekey/pytest')
+def test_template_c_onekey_pytest():
+    templ = qmk.keymap.template_c('handwired/onekey/pytest')
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n'
 
 
-def test_template_onekey_pytest_json():
-    templ = qmk.keymap.template('handwired/onekey/pytest', type='json')
+def test_template_json_onekey_pytest():
+    templ = qmk.keymap.template_json('handwired/onekey/pytest')
     assert templ == {'keyboard': 'handwired/onekey/pytest', "documentation": "This file is a keymap.json file for handwired/onekey/pytest"}
 
 
-def test_generate_onekey_pytest():
-    templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
+def test_generate_c_onekey_pytest():
+    templ = qmk.keymap.generate_c('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n'
 
 
-def test_generate_onekey_pytest_json():
-    templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']], type='json', keymap='default')
+def test_generate_json_onekey_pytest():
+    templ = qmk.keymap.generate_json('default', 'handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
     assert templ == {"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]}