diff options
Diffstat (limited to 'lib/python/qmk/cli')
32 files changed, 1213 insertions, 206 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 10536bb230..d07790d118 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -2,31 +2,159 @@ We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup. """ +import os +import shlex import sys +from importlib.util import find_spec +from pathlib import Path +from subprocess import run -from milc import cli - -from . import c2json -from . import cformat -from . import chibios -from . import clean -from . import compile -from . import config -from . import docs -from . import doctor -from . import flash -from . import generate -from . import hello -from . import info -from . import json -from . import json2c -from . import lint -from . import list -from . import kle2json -from . import new -from . import pyformat -from . import pytest - -if sys.version_info[0] != 3 or sys.version_info[1] < 6: - cli.log.error('Your Python is too old! Please upgrade to Python 3.6 or later.') +from milc import cli, __VERSION__ +from milc.questions import yesno + + +def _run_cmd(*command): + """Run a command in a subshell. + """ + if 'windows' in cli.platform.lower(): + safecmd = map(shlex.quote, command) + safecmd = ' '.join(safecmd) + command = [os.environ['SHELL'], '-c', safecmd] + + return run(command) + + +def _find_broken_requirements(requirements): + """ Check if the modules in the given requirements.txt are available. + + Args: + + requirements + The path to a requirements.txt file + + Returns a list of modules that couldn't be imported + """ + with Path(requirements).open() as fd: + broken_modules = [] + + for line in fd.readlines(): + line = line.strip().replace('<', '=').replace('>', '=') + + if len(line) == 0 or line[0] == '#' or line.startswith('-r'): + continue + + if '#' in line: + line = line.split('#')[0] + + module_name = line.split('=')[0] if '=' in line else line + module_import = module_name.replace('-', '_') + + # Not every module is importable by its own name. + if module_name == "pep8-naming": + module_import = "pep8ext_naming" + + if not find_spec(module_import): + broken_modules.append(module_name) + + return broken_modules + + +def _broken_module_imports(requirements): + """Make sure we can import all the python modules. + """ + broken_modules = _find_broken_requirements(requirements) + + for module in broken_modules: + print('Could not find module %s!' % module) + + if broken_modules: + return True + + return False + + +# Make sure our python is new enough +# +# Supported version information +# +# Based on the OSes we support these are the minimum python version available by default. +# Last update: 2021 Jan 02 +# +# Arch: 3.9 +# Debian: 3.7 +# Fedora 31: 3.7 +# Fedora 32: 3.8 +# Fedora 33: 3.9 +# FreeBSD: 3.7 +# Gentoo: 3.7 +# macOS: 3.9 (from homebrew) +# msys2: 3.8 +# Slackware: 3.7 +# solus: 3.7 +# void: 3.9 + +if sys.version_info[0] != 3 or sys.version_info[1] < 7: + print('Error: Your Python is too old! Please upgrade to Python 3.7 or later.') exit(127) + +milc_version = __VERSION__.split('.') + +if int(milc_version[0]) < 2 and int(milc_version[1]) < 3: + requirements = Path('requirements.txt').resolve() + + print(f'Your MILC library is too old! Please upgrade: python3 -m pip install -U -r {str(requirements)}') + exit(127) + +# Check to make sure we have all our dependencies +msg_install = 'Please run `python3 -m pip install -r %s` to install required python dependencies.' + +if _broken_module_imports('requirements.txt'): + if yesno('Would you like to install the required Python modules?'): + _run_cmd(sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt') + else: + print() + print(msg_install % (str(Path('requirements.txt').resolve()),)) + print() + exit(1) + +if cli.config.user.developer: + args = sys.argv[1:] + while args and args[0][0] == '-': + del args[0] + if not args or args[0] != 'config': + if _broken_module_imports('requirements-dev.txt'): + if yesno('Would you like to install the required developer Python modules?'): + _run_cmd(sys.executable, '-m', 'pip', 'install', '-r', 'requirements-dev.txt') + elif yesno('Would you like to disable developer mode?'): + _run_cmd(sys.argv[0], 'config', 'user.developer=None') + else: + print() + print(msg_install % (str(Path('requirements-dev.txt').resolve()),)) + print('You can also turn off developer mode: qmk config user.developer=None') + print() + exit(1) + +# Import our subcommands +from . import bux # noqa +from . import c2json # noqa +from . import cformat # noqa +from . import chibios # noqa +from . import clean # noqa +from . import compile # noqa +from milc.subcommand import config # noqa +from . import docs # noqa +from . import doctor # noqa +from . import fileformat # noqa +from . import flash # noqa +from . import format # noqa +from . import generate # noqa +from . import hello # noqa +from . import info # noqa +from . import json2c # noqa +from . import lint # noqa +from . import list # noqa +from . import kle2json # noqa +from . import multibuild # noqa +from . import new # noqa +from . import pyformat # noqa +from . import pytest # noqa diff --git a/lib/python/qmk/cli/bux.py b/lib/python/qmk/cli/bux.py new file mode 100755 index 0000000000..504ee35d6e --- /dev/null +++ b/lib/python/qmk/cli/bux.py @@ -0,0 +1,49 @@ +"""QMK Bux + +World domination secret weapon. +""" +from milc import cli +from milc.subcommand import config + + +@cli.subcommand('QMK Bux miner.', hidden=True) +def bux(cli): + """QMK bux + """ + if not cli.config.user.bux: + bux = 0 + else: + bux = cli.config.user.bux + + cli.args.read_only = False + config.set_config('user', 'bux', bux + 1) + cli.save_config() + + buck = """ +@@BBBBBBBBBBBBBBBBBBBBK `vP8#####BE2~ x###g_ `S###q n##} -j#Bl. vBBBBBBBBBBBBBBBBBBBB@@ +@B `:!: ^#@#]- `!t@@&. 7@@B@#^ _Q@Q@@R y@@l:P@#1' `!!_ B@ +@B r@@@B g@@| ` N@@u 7@@iv@@u *#@z"@@R y@@&@@Q- l@@@D B@ +@B !#@B ^#@#x- I@B@@&' 7@@i "B@Q@@r _@@R y@@l.k#@W: `:@@D B@ +@B B@B `v3g#####B0N#d. v##x 'ckk: -##A u##i `lB#I_ @@D B@ +@B B@B @@D B@ +@B B@B `._":!!!=~^*|)r^~:' @@D B@ +@B ~*~ `,=)]}y2tjIIfKfKfaPsffsWsUyx~. **! B@ +@B .*r***r= _*]yzKsqKUfz22IAA3HzzUjtktzHWsHsIz]. B@ +@B )v` , !1- -rysHHUzUzo2jzoI22ztzkyykt2zjzUzIa3qPsl' !r*****` B@ +@B :} @` .j `xzqdAfzKWsj2kkcycczqAsk2zHbg&ER5q55SNN5U~ !RBB#d`c#1 f#\BQ&v B@ +@B _y ]# ,c vUWNWWPsfsssN9WyccnckAfUfWb0DR0&R5RRRddq2_ `@D`jr@2U@#c3@1@Qc- B@ +@B !7! .r]` }AE0RdRqNd9dNR9fUIzzosPqqAddNNdER9EE9dPy! BQ!zy@iU@.Q@@y@8x- B@ +@B :****>. '7adddDdR&gRNdRbd&dNNbbRdNdd5NdRRD0RSf}- .k0&EW`xR .8Q=NRRx B@ +@B =**-rx*r}r~}" ;n2jkzsf3N3zsKsP5dddRddddRddNNqPzy\" '~****" B@ +@B :!!~!;=~r>:*_ `:^vxikylulKfHkyjzzozoIoklix|^!-` B@ +@B ```'-_""::::!:_-.`` B@ +@B `- .` B@ +@B r@= In source we trust @H B@ +@B r@= @H B@ +@B -g@= `}&###E7 W#g. :#Q n####~ R###8k ;#& `##.7#8-`R#z t@H B@ +@B r@= 8@R=-=R@g R@@#:!@@ 2@&!:` 8@1=@@!*@B `@@- v@#8@y @H B@ +@B r@= :@@- _@@_R@fB#}@@ 2@@@# 8@@#@Q.*@B `@@- y@@N @H B@ +@B `. g@9=_~D@g R@}`&@@@ 2@&__` 8@u_Q@2!@@^-x@@` Y@QD@z .` B@ +@@BBBBBBBBBBBBBBBBBBB_ `c8@@@81` S#] `N#B l####v D###BA. vg@@#0~ i#&' 5#K RBBBBBBBBBBBBBBBBBB@@ +""" # noqa: Do not care about the ASCII art + print(f"{buck}\nYou've been blessed by the QMK gods!\nYou have {cli.config.user.bux} QMK bux.") diff --git a/lib/python/qmk/cli/c2json.py b/lib/python/qmk/cli/c2json.py index 2b3bb774f7..43110a9387 100644 --- a/lib/python/qmk/cli/c2json.py +++ b/lib/python/qmk/cli/c2json.py @@ -2,18 +2,22 @@ """ import json +from argcomplete.completers import FilesCompleter from milc import cli import qmk.keymap import qmk.path +from qmk.json_encoders import InfoJSONEncoder +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.errors import CppError @cli.argument('--no-cpp', arg_only=True, action='store_false', help='Do not use \'cpp\' on keymap.c') @cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('-kb', '--keyboard', arg_only=True, required=True, help='The keyboard\'s name') +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name') @cli.argument('-km', '--keymap', arg_only=True, required=True, help='The keymap\'s name') -@cli.argument('filename', arg_only=True, help='keymap.c file') +@cli.argument('filename', arg_only=True, completer=FilesCompleter('.c'), help='keymap.c file') @cli.subcommand('Creates a keymap.json from a keymap.c file.') def c2json(cli): """Generate a keymap.json from a keymap.c file. @@ -34,7 +38,13 @@ def c2json(cli): cli.args.output = None # Parse the keymap.c - keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp) + try: + keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp) + except CppError as e: + if cli.config.general.verbose: + cli.log.debug('The C pre-processor ran into a fatal error: %s', e) + cli.log.error('Something went wrong. Try to use --no-cpp.\nUse the CLI in verbose mode to find out more.') + return False # Generate the keymap.json try: @@ -46,8 +56,8 @@ def c2json(cli): if cli.args.output: cli.args.output.parent.mkdir(parents=True, exist_ok=True) if cli.args.output.exists(): - cli.args.output.replace(cli.args.output.name + '.bak') - cli.args.output.write_text(json.dumps(keymap_json)) + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder)) if not cli.args.quiet: cli.log.info('Wrote keymap to %s.', cli.args.output) diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py index 5aab31843c..efeb459676 100644 --- a/lib/python/qmk/cli/cformat.py +++ b/lib/python/qmk/cli/cformat.py @@ -1,65 +1,137 @@ """Format C code according to QMK's style. """ -import subprocess +from os import path from shutil import which +from subprocess import CalledProcessError, DEVNULL, Popen, PIPE +from argcomplete.completers import FilesCompleter from milc import cli from qmk.path import normpath from qmk.c_parse import c_source_files +c_file_suffixes = ('c', 'h', 'cpp') +core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms') +ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios') -def cformat_run(files, all_files): + +def find_clang_format(): + """Returns the path to clang-format. + """ + for clang_version in range(20, 6, -1): + binary = f'clang-format-{clang_version}' + + if which(binary): + return binary + + return 'clang-format' + + +def find_diffs(files): + """Run clang-format and diff it against a file. + """ + found_diffs = False + + for file in files: + cli.log.debug('Checking for changes in %s', file) + clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True) + diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True) + + if diff.returncode != 0: + print(diff.stdout) + found_diffs = True + + return found_diffs + + +def cformat_run(files): """Spawn clang-format subprocess with proper arguments """ # Determine which version of clang-format to use - clang_format = ['clang-format', '-i'] - for clang_version in [10, 9, 8, 7]: - binary = 'clang-format-%d' % clang_version - if which(binary): - clang_format[0] = binary - break + clang_format = [find_clang_format(), '-i'] + try: - if not files: - cli.log.warn('No changes detected. Use "qmk cformat -a" to format all files') - return False - subprocess.run(clang_format + [file for file in files], check=True) + cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL) cli.log.info('Successfully formatted the C code.') + return True - except subprocess.CalledProcessError: + except CalledProcessError as e: cli.log.error('Error formatting C code!') + cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode) + cli.log.debug('STDOUT:') + cli.log.debug(e.stdout) + cli.log.debug('STDERR:') + cli.log.debug(e.stderr) return False -@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') +def filter_files(files, core_only=False): + """Yield only files to be formatted and skip the rest + """ + if core_only: + # Filter non-core files + for index, file in enumerate(files): + # The following statement checks each file to see if the file path is + # - in the core directories + # - not in the ignored directories + if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): + files[index] = None + cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) + + for file in files: + if file and file.name.split('.')[-1] in c_file_suffixes: + yield file + else: + cli.log.debug('Skipping file %s', file) + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') -@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.') +@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') +@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') +@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') @cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) def cformat(cli): """Format C code according to QMK's style. """ - # Empty array for files - files = [] - # Core directories for formatting - core_dirs = ['drivers', 'quantum', 'tests', 'tmk_core', 'platforms'] - ignores = ['tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios'] # Find the list of files to format if cli.args.files: - files.extend(normpath(file) for file in cli.args.files) + files = list(filter_files(cli.args.files, cli.args.core_only)) + + if not files: + cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) + exit(0) + if cli.args.all_files: cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) - # If -a is specified + elif cli.args.all_files: all_files = c_source_files(core_dirs) - # The following statement checks each file to see if the file path is in the ignored directories. - files.extend(file for file in all_files if not any(i in str(file) for i in ignores)) - # No files specified & no -a flag + files = list(filter_files(all_files, True)) + else: - base_args = ['git', 'diff', '--name-only', cli.args.base_branch] - out = subprocess.run(base_args + core_dirs, check=True, stdout=subprocess.PIPE) - changed_files = filter(None, out.stdout.decode('UTF-8').split('\n')) - filtered_files = [normpath(file) for file in changed_files if not any(i in file for i in ignores)] - files.extend(file for file in filtered_files if file.exists() and file.suffix in ['.c', '.h', '.cpp']) + git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs] + git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) + + if git_diff.returncode != 0: + cli.log.error("Error running %s", git_diff_cmd) + print(git_diff.stderr) + return git_diff.returncode + + files = [] + + for file in git_diff.stdout.strip().split('\n'): + if not any([file.startswith(ignore) for ignore in ignored]): + if path.exists(file) and file.split('.')[-1] in c_file_suffixes: + files.append(file) + + # Sanity check + if not files: + cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files') + return False # Run clang-format on the files we've found - cformat_run(files, cli.args.all_files) + if cli.args.dry_run: + return not find_diffs(files) + else: + return cformat_run(files) diff --git a/lib/python/qmk/cli/chibios/confmigrate.py b/lib/python/qmk/cli/chibios/confmigrate.py index b9cfda9614..be1f2cd744 100644 --- a/lib/python/qmk/cli/chibios/confmigrate.py +++ b/lib/python/qmk/cli/chibios/confmigrate.py @@ -32,7 +32,7 @@ file_header = """\ /* * This file was auto-generated by: - * `qmk chibios-confupdate -i {0} -r {1}` + * `qmk chibios-confmigrate -i {0} -r {1}` */ #pragma once @@ -40,7 +40,7 @@ file_header = """\ def collect_defines(filepath): - with open(filepath, 'r') as f: + with open(filepath, 'r', encoding='utf-8') as f: content = f.read() define_search = re.compile(r'(?m)^#\s*define\s+(?:.*\\\r?\n)*.*$', re.MULTILINE) value_search = re.compile(r'^#\s*define\s+(?P<name>[a-zA-Z0-9_]+(\([^\)]*\))?)\s*(?P<value>.*)', re.DOTALL) @@ -107,10 +107,11 @@ def migrate_mcuconf_h(to_override, outfile): print("", file=outfile) -@cli.argument('-i', '--input', type=normpath, arg_only=True, help='Specify input config file.') -@cli.argument('-r', '--reference', type=normpath, arg_only=True, help='Specify the reference file to compare against') +@cli.argument('-i', '--input', type=normpath, arg_only=True, required=True, help='Specify input config file.') +@cli.argument('-r', '--reference', type=normpath, arg_only=True, required=True, help='Specify the reference file to compare against') @cli.argument('-o', '--overwrite', arg_only=True, action='store_true', help='Overwrites the input file during migration.') @cli.argument('-d', '--delete', arg_only=True, action='store_true', help='If the file has no overrides, migration will delete the input file.') +@cli.argument('-f', '--force', arg_only=True, action='store_true', help='Re-migrates an already migrated file, even if it doesn\'t detect a full ChibiOS config.') @cli.subcommand('Generates a migrated ChibiOS configuration file, as a result of comparing the input against a reference') def chibios_confmigrate(cli): """Generates a usable ChibiOS replacement configuration file, based on a fully-defined conf and a reference config. @@ -142,20 +143,20 @@ def chibios_confmigrate(cli): eprint('--------------------------------------') - if "CHCONF_H" in input_defs["dict"] or "_CHCONF_H_" in input_defs["dict"]: + if cli.args.input.name == "chconf.h" and ("CHCONF_H" in input_defs["dict"] or "_CHCONF_H_" in input_defs["dict"] or cli.args.force): migrate_chconf_h(to_override, outfile=sys.stdout) if cli.args.overwrite: - with open(cli.args.input, "w") as out_file: + with open(cli.args.input, "w", encoding='utf-8') as out_file: migrate_chconf_h(to_override, outfile=out_file) - elif "HALCONF_H" in input_defs["dict"] or "_HALCONF_H_" in input_defs["dict"]: + elif cli.args.input.name == "halconf.h" and ("HALCONF_H" in input_defs["dict"] or "_HALCONF_H_" in input_defs["dict"] or cli.args.force): migrate_halconf_h(to_override, outfile=sys.stdout) if cli.args.overwrite: - with open(cli.args.input, "w") as out_file: + with open(cli.args.input, "w", encoding='utf-8') as out_file: migrate_halconf_h(to_override, outfile=out_file) - elif "MCUCONF_H" in input_defs["dict"] or "_MCUCONF_H_" in input_defs["dict"]: + elif cli.args.input.name == "mcuconf.h" and ("MCUCONF_H" in input_defs["dict"] or "_MCUCONF_H_" in input_defs["dict"] or cli.args.force): migrate_mcuconf_h(to_override, outfile=sys.stdout) if cli.args.overwrite: - with open(cli.args.input, "w") as out_file: + with open(cli.args.input, "w", encoding='utf-8') as out_file: migrate_mcuconf_h(to_override, outfile=out_file) diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py index ec6501b760..72b7ffe810 100644 --- a/lib/python/qmk/cli/clean.py +++ b/lib/python/qmk/cli/clean.py @@ -1,9 +1,9 @@ """Clean the QMK firmware folder of build artifacts. """ -from qmk.commands import run -from milc import cli +from subprocess import DEVNULL -import shutil +from qmk.commands import create_make_target +from milc import cli @cli.argument('-a', '--all', arg_only=True, action='store_true', help='Remove *.hex and *.bin files in the QMK root as well.') @@ -11,6 +11,4 @@ import shutil def clean(cli): """Runs `make clean` (or `make distclean` if --all is passed) """ - make_cmd = 'gmake' if shutil.which('gmake') else 'make' - - run([make_cmd, 'distclean' if cli.args.all else 'clean']) + cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 322ce6a257..7a45e77214 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -2,17 +2,21 @@ You can compile a keymap already in the repo or using a QMK Configurator export. """ -from argparse import FileType +from subprocess import DEVNULL +from argcomplete.completers import FilesCompleter from milc import cli +import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keymap import keymap_completer -@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export to compile') -@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') -@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @@ -29,8 +33,7 @@ def compile(cli): """ if cli.args.clean and not cli.args.filename and not cli.args.dry_run: command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean') - # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere. - cli.run(command, capture_output=False, text=False) + cli.run(command, capture_output=False, stdin=DEVNULL) # Build the environment vars envs = {} diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py index 70f32911a4..9e10570620 100755 --- a/lib/python/qmk/cli/doctor.py +++ b/lib/python/qmk/cli/doctor.py @@ -3,12 +3,12 @@ Check out the user's QMK environment and make sure it's ready to compile. """ import platform +from subprocess import DEVNULL from milc import cli from milc.questions import yesno from qmk import submodules from qmk.constants import QMK_FIRMWARE -from qmk.commands import run from qmk.os_helpers import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo @@ -31,16 +31,27 @@ def os_tests(): def os_test_linux(): """Run the Linux specific tests. """ - cli.log.info("Detected {fg_cyan}Linux.") - from qmk.os_helpers.linux import check_udev_rules + # Don't bother with udev on WSL, for now + if 'microsoft' in platform.uname().release.lower(): + cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.") - return check_udev_rules() + # https://github.com/microsoft/WSL/issues/4197 + if QMK_FIRMWARE.as_posix().startswith("/mnt"): + cli.log.warning("I/O performance on /mnt may be extremely slow.") + return CheckStatus.WARNING + + return CheckStatus.OK + else: + cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") + from qmk.os_helpers.linux import check_udev_rules + + return check_udev_rules() def os_test_macos(): """Run the Mac specific tests. """ - cli.log.info("Detected {fg_cyan}macOS.") + cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) return CheckStatus.OK @@ -48,7 +59,8 @@ def os_test_macos(): def os_test_windows(): """Run the Windows specific tests. """ - cli.log.info("Detected {fg_cyan}Windows.") + win32_ver = platform.win32_ver() + cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1]) return CheckStatus.OK @@ -65,11 +77,10 @@ def doctor(cli): * [ ] Compile a trivial program with each compiler """ cli.log.info('QMK Doctor is checking your environment.') + cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE) status = os_tests() - cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE) - # Make sure our QMK home is a Git repo git_ok = check_git_repo() @@ -82,7 +93,7 @@ def doctor(cli): if not bin_ok: if yesno('Would you like to install dependencies?', default=True): - run(['util/qmk_install.sh']) + cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False) bin_ok = check_binaries() if bin_ok: @@ -107,9 +118,9 @@ def doctor(cli): submodules.update() sub_ok = check_submodules() - if CheckStatus.ERROR in sub_ok: + if sub_ok == CheckStatus.ERROR: status = CheckStatus.ERROR - elif CheckStatus.WARNING in sub_ok and status == CheckStatus.OK: + elif sub_ok == CheckStatus.WARNING and status == CheckStatus.OK: status = CheckStatus.WARNING # Report a summary of our findings to the user diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index a876290035..1b2932a5b2 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -3,13 +3,15 @@ You can compile a keymap already in the repo or using a QMK Configurator export. A bootloader must be specified. """ -from argparse import FileType +from subprocess import DEVNULL +from argcomplete.completers import FilesCompleter from milc import cli import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.keyboard import keyboard_completer, keyboard_folder def print_bootloader_help(): @@ -30,11 +32,11 @@ def print_bootloader_help(): cli.echo('For more info, visit https://docs.qmk.fm/#/flashing') -@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), help='The configurator export JSON to compile.') +@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export JSON to compile.') @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.') @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') -@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @@ -54,7 +56,7 @@ def flash(cli): """ if cli.args.clean and not cli.args.filename and not cli.args.dry_run: command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean') - cli.run(command, capture_output=False) + cli.run(command, capture_output=False, stdin=DEVNULL) # Build the environment vars envs = {} @@ -97,7 +99,7 @@ def flash(cli): cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) if not cli.args.dry_run: cli.echo('\n') - compile = cli.run(command, capture_output=False, text=True) + compile = cli.run(command, capture_output=False, stdin=DEVNULL) return compile.returncode else: diff --git a/lib/python/qmk/cli/format/__init__.py b/lib/python/qmk/cli/format/__init__.py new file mode 100644 index 0000000000..741ec778b1 --- /dev/null +++ b/lib/python/qmk/cli/format/__init__.py @@ -0,0 +1 @@ +from . import json diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py new file mode 100755 index 0000000000..1358c70e7a --- /dev/null +++ b/lib/python/qmk/cli/format/json.py @@ -0,0 +1,66 @@ +"""JSON Formatting Script + +Spits out a JSON file formatted with one of QMK's formatters. +""" +import json + +from jsonschema import ValidationError +from milc import cli + +from qmk.info import info_json +from qmk.json_schema import json_load, keyboard_validate +from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder +from qmk.path import normpath + + +@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') +@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') +@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) +def format_json(cli): + """Format a json file. + """ + json_file = json_load(cli.args.json_file) + + if cli.args.format == 'auto': + try: + keyboard_validate(json_file) + json_encoder = InfoJSONEncoder + + except ValidationError as e: + cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) + cli.log.info('Treating %s as a keymap file.', cli.args.json_file) + json_encoder = KeymapJSONEncoder + + elif cli.args.format == 'keyboard': + json_encoder = InfoJSONEncoder + elif cli.args.format == 'keymap': + json_encoder = KeymapJSONEncoder + else: + # This should be impossible + cli.log.error('Unknown format: %s', cli.args.format) + return False + + if json_encoder == KeymapJSONEncoder and 'layout' in json_file: + # Attempt to format the keycodes. + layout = json_file['layout'] + info_data = info_json(json_file['keyboard']) + + if layout in info_data.get('layout_aliases', {}): + layout = json_file['layout'] = info_data['layout_aliases'][layout] + + if layout in info_data.get('layouts'): + for layer_num, layer in enumerate(json_file['layers']): + current_layer = [] + last_row = 0 + + for keymap_key, info_key in zip(layer, info_data['layouts'][layout]['layout']): + if last_row != info_key['y']: + current_layer.append('JSON_NEWLINE') + last_row = info_key['y'] + + current_layer.append(keymap_key) + + json_file['layers'][layer_num] = current_layer + + # Display the results + print(json.dumps(json_file, cls=json_encoder)) diff --git a/lib/python/qmk/cli/generate/__init__.py b/lib/python/qmk/cli/generate/__init__.py index f9585bfb5c..0efca0022d 100644 --- a/lib/python/qmk/cli/generate/__init__.py +++ b/lib/python/qmk/cli/generate/__init__.py @@ -1,3 +1,9 @@ from . import api +from . import config_h +from . import dfu_header from . import docs +from . import info_json +from . import keyboard_h +from . import layouts from . import rgb_breathe_table +from . import rules_mk diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py index 66db37cb52..285bd90eb5 100755 --- a/lib/python/qmk/cli/generate/api.py +++ b/lib/python/qmk/cli/generate/api.py @@ -8,51 +8,80 @@ from milc import cli from qmk.datetime import current_datetime from qmk.info import info_json -from qmk.keyboard import list_keyboards +from qmk.json_encoders import InfoJSONEncoder +from qmk.json_schema import json_load +from qmk.keyboard import find_readme, list_keyboards +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.") @cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True) def generate_api(cli): """Generates the QMK API data. """ api_data_dir = Path('api_data') v1_dir = api_data_dir / 'v1' - keyboard_list = v1_dir / 'keyboard_list.json' - keyboard_all = v1_dir / 'keyboards.json' - usb_file = v1_dir / 'usb.json' + keyboard_all_file = v1_dir / 'keyboards.json' # A massive JSON containing everything + keyboard_list_file = v1_dir / 'keyboard_list.json' # A simple list of keyboard targets + keyboard_aliases_file = v1_dir / 'keyboard_aliases.json' # A list of historical keyboard names and their new name + keyboard_metadata_file = v1_dir / 'keyboard_metadata.json' # All the data configurator/via needs for initialization + usb_file = v1_dir / 'usb.json' # A mapping of USB VID/PID -> keyboard target if not api_data_dir.exists(): api_data_dir.mkdir() - kb_all = {'last_updated': current_datetime(), 'keyboards': {}} - usb_list = {'last_updated': current_datetime(), 'devices': {}} + kb_all = {} + usb_list = {} # Generate and write keyboard specific JSON files for keyboard_name in list_keyboards(): - kb_all['keyboards'][keyboard_name] = info_json(keyboard_name) + kb_all[keyboard_name] = info_json(keyboard_name) keyboard_dir = v1_dir / 'keyboards' / keyboard_name keyboard_info = keyboard_dir / 'info.json' keyboard_readme = keyboard_dir / 'readme.md' - keyboard_readme_src = Path('keyboards') / keyboard_name / 'readme.md' + keyboard_readme_src = find_readme(keyboard_name) keyboard_dir.mkdir(parents=True, exist_ok=True) - keyboard_info.write_text(json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_all['keyboards'][keyboard_name]}})) + keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_all[keyboard_name]}}) + if not cli.args.dry_run: + keyboard_info.write_text(keyboard_json) + cli.log.debug('Wrote file %s', keyboard_info) - if keyboard_readme_src.exists(): - copyfile(keyboard_readme_src, keyboard_readme) + if keyboard_readme_src: + copyfile(keyboard_readme_src, keyboard_readme) + cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme) - if 'usb' in kb_all['keyboards'][keyboard_name]: - usb = kb_all['keyboards'][keyboard_name]['usb'] + if 'usb' in kb_all[keyboard_name]: + usb = kb_all[keyboard_name]['usb'] - if usb['vid'] not in usb_list['devices']: - usb_list['devices'][usb['vid']] = {} + if 'vid' in usb and usb['vid'] not in usb_list: + usb_list[usb['vid']] = {} - if usb['pid'] not in usb_list['devices'][usb['vid']]: - usb_list['devices'][usb['vid']][usb['pid']] = {} + if 'pid' in usb and usb['pid'] not in usb_list[usb['vid']]: + usb_list[usb['vid']][usb['pid']] = {} - usb_list['devices'][usb['vid']][usb['pid']][keyboard_name] = usb + if 'vid' in usb and 'pid' in usb: + usb_list[usb['vid']][usb['pid']][keyboard_name] = usb + + # Generate data for the global files + keyboard_list = sorted(kb_all) + keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.json')) + keyboard_metadata = { + 'last_updated': current_datetime(), + 'keyboards': keyboard_list, + 'keyboard_aliases': keyboard_aliases, + 'usb': usb_list, + } # Write the global JSON files - keyboard_list.write_text(json.dumps({'last_updated': current_datetime(), 'keyboards': sorted(kb_all['keyboards'])})) - keyboard_all.write_text(json.dumps(kb_all)) - usb_file.write_text(json.dumps(usb_list)) + keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, cls=InfoJSONEncoder) + usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, cls=InfoJSONEncoder) + keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, cls=InfoJSONEncoder) + keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, cls=InfoJSONEncoder) + keyboard_metadata_json = json.dumps(keyboard_metadata, cls=InfoJSONEncoder) + + if not cli.args.dry_run: + keyboard_all_file.write_text(keyboard_all_json) + usb_file.write_text(usb_json) + keyboard_list_file.write_text(keyboard_list_json) + keyboard_aliases_file.write_text(keyboard_aliases_json) + keyboard_metadata_file.write_text(keyboard_metadata_json) diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py new file mode 100755 index 0000000000..54cd5b96a8 --- /dev/null +++ b/lib/python/qmk/cli/generate/config_h.py @@ -0,0 +1,154 @@ +"""Used by the make system to generate info_config.h from info.json. +""" +from pathlib import Path + +from dotty_dict import dotty +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.info import info_json +from qmk.json_schema import json_load +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.path import is_keyboard, normpath + + +def direct_pins(direct_pins): + """Return the config.h lines that set the direct pins. + """ + rows = [] + + for row in direct_pins: + cols = ','.join(map(str, [col or 'NO_PIN' for col in row])) + rows.append('{' + cols + '}') + + col_count = len(direct_pins[0]) + row_count = len(direct_pins) + + return """ +#ifndef MATRIX_COLS +# define MATRIX_COLS %s +#endif // MATRIX_COLS + +#ifndef MATRIX_ROWS +# define MATRIX_ROWS %s +#endif // MATRIX_ROWS + +#ifndef DIRECT_PINS +# define DIRECT_PINS {%s} +#endif // DIRECT_PINS +""" % (col_count, row_count, ','.join(rows)) + + +def pin_array(define, pins): + """Return the config.h lines that set a pin array. + """ + pin_num = len(pins) + pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins])) + + return f""" +#ifndef {define}S +# define {define}S {pin_num} +#endif // {define}S + +#ifndef {define}_PINS +# define {define}_PINS {{ {pin_array} }} +#endif // {define}_PINS +""" + + +def matrix_pins(matrix_pins): + """Add the matrix config to the config.h. + """ + pins = [] + + if 'direct' in matrix_pins: + pins.append(direct_pins(matrix_pins['direct'])) + + if 'cols' in matrix_pins: + pins.append(pin_array('MATRIX_COL', matrix_pins['cols'])) + + if 'rows' in matrix_pins: + pins.append(pin_array('MATRIX_ROW', matrix_pins['rows'])) + + return '\n'.join(pins) + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') +@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) +@automagic_keyboard +@automagic_keymap +def generate_config_h(cli): + """Generates the info_config.h file. + """ + # Determine our keyboard(s) + if not cli.config.generate_config_h.keyboard: + cli.log.error('Missing parameter: --keyboard') + cli.subcommands['info'].print_help() + return False + + if not is_keyboard(cli.config.generate_config_h.keyboard): + cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard) + return False + + # Build the info_config.h file. + kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard)) + info_config_map = json_load(Path('data/mappings/info_config.json')) + + config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once'] + + # Iterate through the info_config map to generate basic things + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + to_config = info_dict.get('to_config', True) + + if not to_config: + continue + + try: + config_value = kb_info_json[info_key] + except KeyError: + continue + + if key_type.startswith('array'): + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key} {{ {", ".join(map(str, config_value))} }}') + config_h_lines.append(f'#endif // {config_key}') + elif key_type == 'bool': + if config_value: + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key}') + config_h_lines.append(f'#endif // {config_key}') + elif key_type == 'mapping': + for key, value in config_value.items(): + config_h_lines.append('') + config_h_lines.append(f'#ifndef {key}') + config_h_lines.append(f'# define {key} {value}') + config_h_lines.append(f'#endif // {key}') + else: + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key} {config_value}') + config_h_lines.append(f'#endif // {config_key}') + + if 'matrix_pins' in kb_info_json: + config_h_lines.append(matrix_pins(kb_info_json['matrix_pins'])) + + # Show the results + config_h = '\n'.join(config_h_lines) + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + if cli.args.output.exists(): + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(config_h) + + if not cli.args.quiet: + cli.log.info('Wrote info_config.h to %s.', cli.args.output) + + else: + print(config_h) diff --git a/lib/python/qmk/cli/generate/dfu_header.py b/lib/python/qmk/cli/generate/dfu_header.py new file mode 100644 index 0000000000..211ed9991a --- /dev/null +++ b/lib/python/qmk/cli/generate/dfu_header.py @@ -0,0 +1,60 @@ +"""Used by the make system to generate LUFA Keyboard.h from info.json +""" +from dotty_dict import dotty +from milc import cli + +from qmk.decorators import automagic_keyboard +from qmk.info import info_json +from qmk.path import is_keyboard, normpath +from qmk.keyboard import keyboard_completer + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='Keyboard to generate LUFA Keyboard.h for.') +@cli.subcommand('Used by the make system to generate LUFA Keyboard.h from info.json', hidden=True) +@automagic_keyboard +def generate_dfu_header(cli): + """Generates the Keyboard.h file. + """ + # Determine our keyboard(s) + if not cli.config.generate_dfu_header.keyboard: + cli.log.error('Missing parameter: --keyboard') + cli.subcommands['info'].print_help() + return False + + if not is_keyboard(cli.config.generate_dfu_header.keyboard): + cli.log.error('Invalid keyboard: "%s"', cli.config.generate_dfu_header.keyboard) + return False + + # Build the Keyboard.h file. + kb_info_json = dotty(info_json(cli.config.generate_dfu_header.keyboard)) + + keyboard_h_lines = ['/* This file was generated by `qmk generate-dfu-header`. Do not edit or copy.' ' */', '', '#pragma once'] + keyboard_h_lines.append(f'#define MANUFACTURER {kb_info_json["manufacturer"]}') + keyboard_h_lines.append(f'#define PRODUCT {cli.config.generate_dfu_header.keyboard} Bootloader') + + # Optional + if 'qmk_lufa_bootloader.esc_output' in kb_info_json: + keyboard_h_lines.append(f'#define QMK_ESC_OUTPUT {kb_info_json["qmk_lufa_bootloader.esc_output"]}') + if 'qmk_lufa_bootloader.esc_input' in kb_info_json: + keyboard_h_lines.append(f'#define QMK_ESC_INPUT {kb_info_json["qmk_lufa_bootloader.esc_input"]}') + if 'qmk_lufa_bootloader.led' in kb_info_json: + keyboard_h_lines.append(f'#define QMK_LED {kb_info_json["qmk_lufa_bootloader.led"]}') + if 'qmk_lufa_bootloader.speaker' in kb_info_json: + keyboard_h_lines.append(f'#define QMK_SPEAKER {kb_info_json["qmk_lufa_bootloader.speaker"]}') + + # Show the results + keyboard_h = '\n'.join(keyboard_h_lines) + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + if cli.args.output.exists(): + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(keyboard_h) + + if not cli.args.quiet: + cli.log.info('Wrote Keyboard.h to %s.', cli.args.output) + + else: + print(keyboard_h) diff --git a/lib/python/qmk/cli/generate/docs.py b/lib/python/qmk/cli/generate/docs.py index a59a24db50..749336fea5 100644 --- a/lib/python/qmk/cli/generate/docs.py +++ b/lib/python/qmk/cli/generate/docs.py @@ -1,8 +1,8 @@ """Build QMK documentation locally """ import shutil -import subprocess from pathlib import Path +from subprocess import DEVNULL from milc import cli @@ -24,14 +24,16 @@ def generate_docs(cli): shutil.copytree(DOCS_PATH, BUILD_PATH) # When not verbose we want to hide all output - args = {'check': True} - if not cli.args.verbose: - args.update({'stdout': subprocess.DEVNULL, 'stderr': subprocess.STDOUT}) + args = { + 'capture_output': False if cli.config.general.verbose else True, + 'check': True, + 'stdin': DEVNULL, + } cli.log.info('Generating internal docs...') # Generate internal docs - subprocess.run(['doxygen', 'Doxyfile'], **args) - subprocess.run(['moxygen', '-q', '-a', '-g', '-o', BUILD_PATH / 'internals_%s.md', 'doxygen/xml'], **args) + cli.run(['doxygen', 'Doxyfile'], **args) + cli.run(['moxygen', '-q', '-a', '-g', '-o', BUILD_PATH / 'internals_%s.md', 'doxygen/xml'], **args) cli.log.info('Successfully generated internal docs to %s.', BUILD_PATH) diff --git a/lib/python/qmk/cli/generate/info_json.py b/lib/python/qmk/cli/generate/info_json.py new file mode 100755 index 0000000000..8931b68b6f --- /dev/null +++ b/lib/python/qmk/cli/generate/info_json.py @@ -0,0 +1,67 @@ +"""Keyboard information script. + +Compile an info.json for a particular keyboard and pretty-print it. +""" +import json + +from jsonschema import Draft7Validator, validators +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.info import info_json +from qmk.json_encoders import InfoJSONEncoder +from qmk.json_schema import load_jsonschema +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.path import is_keyboard + + +def pruning_validator(validator_class): + """Extends Draft7Validator to remove properties that aren't specified in the schema. + """ + validate_properties = validator_class.VALIDATORS["properties"] + + def remove_additional_properties(validator, properties, instance, schema): + for prop in list(instance.keys()): + if prop not in properties: + del instance[prop] + + for error in validate_properties(validator, properties, instance, schema): + yield error + + return validators.extend(validator_class, {"properties": remove_additional_properties}) + + +def strip_info_json(kb_info_json): + """Remove the API-only properties from the info.json. + """ + pruning_draft_7_validator = pruning_validator(Draft7Validator) + schema = load_jsonschema('keyboard') + validator = pruning_draft_7_validator(schema).validate + + return validator(kb_info_json) + + +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to show info for.') +@cli.argument('-km', '--keymap', help='Show the layers for a JSON keymap too.') +@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) +@automagic_keyboard +@automagic_keymap +def generate_info_json(cli): + """Generate an info.json file for a keyboard + """ + # Determine our keyboard(s) + if not cli.config.generate_info_json.keyboard: + cli.log.error('Missing parameter: --keyboard') + cli.subcommands['info'].print_help() + return False + + if not is_keyboard(cli.config.generate_info_json.keyboard): + cli.log.error('Invalid keyboard: "%s"', cli.config.generate_info_json.keyboard) + return False + + # Build the info.json file + kb_info_json = info_json(cli.config.generate_info_json.keyboard) + strip_info_json(kb_info_json) + + # Display the results + print(json.dumps(kb_info_json, indent=2, cls=InfoJSONEncoder)) diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py new file mode 100755 index 0000000000..22500dbc91 --- /dev/null +++ b/lib/python/qmk/cli/generate/keyboard_h.py @@ -0,0 +1,60 @@ +"""Used by the make system to generate keyboard.h from info.json. +""" +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.info import info_json +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.path import normpath + + +def would_populate_layout_h(keyboard): + """Detect if a given keyboard is doing data driven layouts + """ + # Build the info.json file + kb_info_json = info_json(keyboard) + + for layout_name in kb_info_json['layouts']: + if kb_info_json['layouts'][layout_name]['c_macro']: + continue + + if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]: + cli.log.debug('%s/%s: No matrix data!', keyboard, layout_name) + continue + + return True + + return False + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.') +@cli.subcommand('Used by the make system to generate keyboard.h from info.json', hidden=True) +@automagic_keyboard +@automagic_keymap +def generate_keyboard_h(cli): + """Generates the keyboard.h file. + """ + has_layout_h = would_populate_layout_h(cli.config.generate_keyboard_h.keyboard) + + # Build the layouts.h file. + keyboard_h_lines = ['/* This file was generated by `qmk generate-keyboard-h`. Do not edit or copy.' ' */', '', '#pragma once', '#include "quantum.h"'] + + if not has_layout_h: + keyboard_h_lines.append('#pragma error("<keyboard>.h is only optional for data driven keyboards - kb.h == bad times")') + + # Show the results + keyboard_h = '\n'.join(keyboard_h_lines) + '\n' + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + if cli.args.output.exists(): + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(keyboard_h) + + if not cli.args.quiet: + cli.log.info('Wrote keyboard_h to %s.', cli.args.output) + + else: + print(keyboard_h) diff --git a/lib/python/qmk/cli/generate/layouts.py b/lib/python/qmk/cli/generate/layouts.py new file mode 100755 index 0000000000..ad6946d6cf --- /dev/null +++ b/lib/python/qmk/cli/generate/layouts.py @@ -0,0 +1,103 @@ +"""Used by the make system to generate layouts.h from info.json. +""" +from milc import cli + +from qmk.constants import COL_LETTERS, ROW_LETTERS +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.info import info_json +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.path import is_keyboard, normpath + +usb_properties = { + 'vid': 'VENDOR_ID', + 'pid': 'PRODUCT_ID', + 'device_ver': 'DEVICE_VER', +} + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') +@cli.subcommand('Used by the make system to generate layouts.h from info.json', hidden=True) +@automagic_keyboard +@automagic_keymap +def generate_layouts(cli): + """Generates the layouts.h file. + """ + # Determine our keyboard(s) + if not cli.config.generate_layouts.keyboard: + cli.log.error('Missing parameter: --keyboard') + cli.subcommands['info'].print_help() + return False + + if not is_keyboard(cli.config.generate_layouts.keyboard): + cli.log.error('Invalid keyboard: "%s"', cli.config.generate_layouts.keyboard) + return False + + # Build the info.json file + kb_info_json = info_json(cli.config.generate_layouts.keyboard) + + # Build the layouts.h file. + layouts_h_lines = ['/* This file was generated by `qmk generate-layouts`. Do not edit or copy.' ' */', '', '#pragma once'] + + if 'matrix_pins' in kb_info_json: + if 'direct' in kb_info_json['matrix_pins']: + col_num = len(kb_info_json['matrix_pins']['direct'][0]) + row_num = len(kb_info_json['matrix_pins']['direct']) + elif 'cols' in kb_info_json['matrix_pins'] and 'rows' in kb_info_json['matrix_pins']: + col_num = len(kb_info_json['matrix_pins']['cols']) + row_num = len(kb_info_json['matrix_pins']['rows']) + else: + cli.log.error('%s: Invalid matrix config.', cli.config.generate_layouts.keyboard) + return False + + for layout_name in kb_info_json['layouts']: + if kb_info_json['layouts'][layout_name]['c_macro']: + continue + + if 'matrix' not in kb_info_json['layouts'][layout_name]['layout'][0]: + cli.log.debug('%s/%s: No matrix data!', cli.config.generate_layouts.keyboard, layout_name) + continue + + layout_keys = [] + layout_matrix = [['KC_NO' for i in range(col_num)] for i in range(row_num)] + + for i, key in enumerate(kb_info_json['layouts'][layout_name]['layout']): + row = key['matrix'][0] + col = key['matrix'][1] + identifier = 'k%s%s' % (ROW_LETTERS[row], COL_LETTERS[col]) + + try: + layout_matrix[row][col] = identifier + layout_keys.append(identifier) + except IndexError: + key_name = key.get('label', identifier) + cli.log.error('Matrix data out of bounds for layout %s at index %s (%s): %s, %s', layout_name, i, key_name, row, col) + return False + + layouts_h_lines.append('') + layouts_h_lines.append('#define %s(%s) {\\' % (layout_name, ', '.join(layout_keys))) + + rows = ', \\\n'.join(['\t {' + ', '.join(row) + '}' for row in layout_matrix]) + rows += ' \\' + layouts_h_lines.append(rows) + layouts_h_lines.append('}') + + for alias, target in kb_info_json.get('layout_aliases', {}).items(): + layouts_h_lines.append('') + layouts_h_lines.append('#define %s %s' % (alias, target)) + + # Show the results + layouts_h = '\n'.join(layouts_h_lines) + '\n' + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + if cli.args.output.exists(): + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(layouts_h) + + if not cli.args.quiet: + cli.log.info('Wrote info_config.h to %s.', cli.args.output) + + else: + print(layouts_h) diff --git a/lib/python/qmk/cli/generate/rgb_breathe_table.py b/lib/python/qmk/cli/generate/rgb_breathe_table.py index e1c5423ee5..7382abd68b 100644 --- a/lib/python/qmk/cli/generate/rgb_breathe_table.py +++ b/lib/python/qmk/cli/generate/rgb_breathe_table.py @@ -70,7 +70,7 @@ static const int table_scale = 256 / sizeof(rgblight_effect_breathe_table); if cli.args.output: cli.args.output.parent.mkdir(parents=True, exist_ok=True) if cli.args.output.exists(): - cli.args.output.replace(cli.args.output.name + '.bak') + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) cli.args.output.write_text(table_template) if not cli.args.quiet: diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py new file mode 100755 index 0000000000..41c94e16b5 --- /dev/null +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -0,0 +1,97 @@ +"""Used by the make system to generate a rules.mk +""" +from pathlib import Path + +from dotty_dict import dotty +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.info import info_json +from qmk.json_schema import json_load +from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.path import is_keyboard, normpath + + +def process_mapping_rule(kb_info_json, rules_key, info_dict): + """Return the rules.mk line(s) for a mapping rule. + """ + if not info_dict.get('to_c', True): + return None + + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + rules_value = kb_info_json[info_key] + except KeyError: + return None + + if key_type == 'array': + return f'{rules_key} ?= {" ".join(rules_value)}' + elif key_type == 'bool': + return f'{rules_key} ?= {"on" if rules_value else "off"}' + elif key_type == 'mapping': + return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()]) + + return f'{rules_key} ?= {rules_value}' + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-e', '--escape', arg_only=True, action='store_true', help="Escape spaces in quiet mode") +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') +@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) +@automagic_keyboard +@automagic_keymap +def generate_rules_mk(cli): + """Generates a rules.mk file from info.json. + """ + if not cli.config.generate_rules_mk.keyboard: + cli.log.error('Missing parameter: --keyboard') + cli.subcommands['info'].print_help() + return False + + if not is_keyboard(cli.config.generate_rules_mk.keyboard): + cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard) + return False + + kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard)) + info_rules_map = json_load(Path('data/mappings/info_rules.json')) + rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', ''] + + # Iterate through the info_rules map to generate basic rules + for rules_key, info_dict in info_rules_map.items(): + new_entry = process_mapping_rule(kb_info_json, rules_key, info_dict) + + if new_entry: + rules_mk_lines.append(new_entry) + + # Iterate through features to enable/disable them + if 'features' in kb_info_json: + for feature, enabled in kb_info_json['features'].items(): + if feature == 'bootmagic_lite' and enabled: + rules_mk_lines.append('BOOTMAGIC_ENABLE ?= lite') + else: + feature = feature.upper() + enabled = 'yes' if enabled else 'no' + rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') + + # Show the results + rules_mk = '\n'.join(rules_mk_lines) + '\n' + + if cli.args.output: + cli.args.output.parent.mkdir(parents=True, exist_ok=True) + if cli.args.output.exists(): + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) + cli.args.output.write_text(rules_mk) + + if cli.args.quiet: + if cli.args.escape: + print(cli.args.output.as_posix().replace(' ', '\\ ')) + else: + print(cli.args.output) + else: + cli.log.info('Wrote rules.mk to %s.', cli.args.output) + + else: + print(rules_mk) diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py index 9ab299a21e..0d08d242cd 100755 --- a/lib/python/qmk/cli/info.py +++ b/lib/python/qmk/cli/info.py @@ -2,21 +2,20 @@ Compile an info.json for a particular keyboard and pretty-print it. """ +import sys import json -import platform from milc import cli +from qmk.json_encoders import InfoJSONEncoder +from qmk.constants import COL_LETTERS, ROW_LETTERS from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.keyboard import render_layouts, render_layout +from qmk.keyboard import keyboard_completer, keyboard_folder, render_layouts, render_layout, rules_mk from qmk.keymap import locate_keymap from qmk.info import info_json from qmk.path import is_keyboard -platform_id = platform.platform().lower() - -ROW_LETTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop' -COL_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijilmnopqrstuvwxyz' +UNICODE_SUPPORT = sys.stdout.encoding.lower().startswith('utf') def show_keymap(kb_info_json, title_caps=True): @@ -30,7 +29,7 @@ def show_keymap(kb_info_json, title_caps=True): else: cli.echo('{fg_blue}keymap_%s{fg_reset}:', cli.config.info.keymap) - keymap_data = json.load(keymap_path.open()) + keymap_data = json.load(keymap_path.open(encoding='utf-8')) layout_name = keymap_data['layout'] for layer_num, layer in enumerate(keymap_data['layers']): @@ -58,7 +57,7 @@ def show_matrix(kb_info_json, title_caps=True): # Build our label list labels = [] for key in layout['layout']: - if key['matrix']: + if 'matrix' in key: row = ROW_LETTERS[key['matrix'][0]] col = COL_LETTERS[key['matrix'][1]] @@ -92,6 +91,9 @@ def print_friendly_output(kb_info_json): cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height'])) cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown')) cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown')) + if 'layout_aliases' in kb_info_json: + aliases = [f'{key}={value}' for key, value in kb_info_json['layout_aliases'].items()] + cli.echo('{fg_blue}Layout aliases:{fg_reset} %s' % (', '.join(aliases),)) if cli.config.info.layouts: show_layouts(kb_info_json, True) @@ -122,12 +124,20 @@ def print_text_output(kb_info_json): show_keymap(kb_info_json, False) -@cli.argument('-kb', '--keyboard', help='Keyboard to show info for.') +def print_parsed_rules_mk(keyboard_name): + rules = rules_mk(keyboard_name) + for k in sorted(rules.keys()): + print('%s = %s' % (k, rules[k])) + return + + +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to show info for.') @cli.argument('-km', '--keymap', help='Show the layers for a JSON keymap too.') @cli.argument('-l', '--layouts', action='store_true', help='Render the layouts.') @cli.argument('-m', '--matrix', action='store_true', help='Render the layouts with matrix information.') @cli.argument('-f', '--format', default='friendly', arg_only=True, help='Format to display the data in (friendly, text, json) (Default: friendly).') -@cli.argument('--ascii', action='store_true', default='windows' in platform_id, help='Render layout box drawings in ASCII only.') +@cli.argument('--ascii', action='store_true', default=not UNICODE_SUPPORT, help='Render layout box drawings in ASCII only.') +@cli.argument('-r', '--rules-mk', action='store_true', help='Render the parsed values of the keyboard\'s rules.mk file.') @cli.subcommand('Keyboard information.') @automagic_keyboard @automagic_keymap @@ -144,12 +154,16 @@ def info(cli): cli.log.error('Invalid keyboard: "%s"', cli.config.info.keyboard) return False + if bool(cli.args.rules_mk): + print_parsed_rules_mk(cli.config.info.keyboard) + return False + # Build the info.json file kb_info_json = info_json(cli.config.info.keyboard) # Output in the requested format if cli.args.format == 'json': - print(json.dumps(kb_info_json)) + print(json.dumps(kb_info_json, cls=InfoJSONEncoder)) elif cli.args.format == 'text': print_text_output(kb_info_json) elif cli.args.format == 'friendly': diff --git a/lib/python/qmk/cli/json2c.py b/lib/python/qmk/cli/json2c.py index 97d8fb0c33..a90578c021 100755 --- a/lib/python/qmk/cli/json2c.py +++ b/lib/python/qmk/cli/json2c.py @@ -1,8 +1,8 @@ """Generate a keymap.c from a configurator export. """ import json -import sys +from argcomplete.completers import FilesCompleter from milc import cli import qmk.keymap @@ -11,7 +11,7 @@ import qmk.path @cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('filename', type=qmk.path.normpath, arg_only=True, help='Configurator JSON file') +@cli.argument('filename', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file') @cli.subcommand('Creates a keymap.c from a QMK Configurator export.') def json2c(cli): """Generate a keymap.c from a configurator export. @@ -20,19 +20,8 @@ def json2c(cli): """ try: - # Parse the configurator from stdin - if cli.args.filename and cli.args.filename.name == '-': - user_keymap = json.load(sys.stdin) - - else: - # Error checking - if not cli.args.filename.exists(): - cli.log.error('JSON file does not exist!') - return False - - # Parse the configurator json file - else: - user_keymap = json.loads(cli.args.filename.read_text()) + # Parse the configurator from json file (or stdin) + user_keymap = json.load(cli.args.filename) except json.decoder.JSONDecodeError as ex: cli.log.error('The JSON input does not appear to be valid.') @@ -49,7 +38,7 @@ def json2c(cli): if cli.args.output: cli.args.output.parent.mkdir(parents=True, exist_ok=True) if cli.args.output.exists(): - cli.args.output.replace(cli.args.output.name + '.bak') + cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) cli.args.output.write_text(keymap_c) if not cli.args.quiet: diff --git a/lib/python/qmk/cli/kle2json.py b/lib/python/qmk/cli/kle2json.py index 3d1bb8c43c..acb75ef4fd 100755 --- a/lib/python/qmk/cli/kle2json.py +++ b/lib/python/qmk/cli/kle2json.py @@ -3,28 +3,16 @@ import json import os from pathlib import Path -from decimal import Decimal -from collections import OrderedDict +from argcomplete.completers import FilesCompleter from milc import cli from kle2xy import KLE2xy from qmk.converter import kle2qmk +from qmk.json_encoders import InfoJSONEncoder -class CustomJSONEncoder(json.JSONEncoder): - def default(self, obj): - try: - if isinstance(obj, Decimal): - if obj % 2 in (Decimal(0), Decimal(1)): - return int(obj) - return float(obj) - except TypeError: - pass - return json.JSONEncoder.default(self, obj) - - -@cli.argument('filename', help='The KLE raw txt to convert') +@cli.argument('filename', completer=FilesCompleter('.json'), help='The KLE raw txt to convert') @cli.argument('-f', '--force', action='store_true', help='Flag to overwrite current info.json') @cli.subcommand('Convert a KLE layout to a Configurator JSON', hidden=False if cli.config.user.developer else True) def kle2json(cli): @@ -40,7 +28,7 @@ def kle2json(cli): cli.log.error('File {fg_cyan}%s{style_reset_all} was not found.', file_path) return False out_path = file_path.parent - raw_code = file_path.open().read() + raw_code = file_path.read_text(encoding='utf-8') # Check if info.json exists, allow overwrite with force if Path(out_path, "info.json").exists() and not cli.args.force: cli.log.error('File {fg_cyan}%s/info.json{style_reset_all} already exists, use -f or --force to overwrite.', out_path) @@ -52,24 +40,22 @@ def kle2json(cli): cli.log.error('Could not parse KLE raw data: %s', raw_code) cli.log.exception(e) return False - keyboard = OrderedDict( - keyboard_name=kle.name, - url='', - maintainer='qmk', - width=kle.columns, - height=kle.rows, - layouts={'LAYOUT': { - 'layout': 'LAYOUT_JSON_HERE' - }}, - ) - # Initialize keyboard with json encoded from ordered dict - keyboard = json.dumps(keyboard, indent=4, separators=(', ', ': '), sort_keys=False, cls=CustomJSONEncoder) - # Initialize layout with kle2qmk from converter module - layout = json.dumps(kle2qmk(kle), separators=(', ', ':'), cls=CustomJSONEncoder) - # Replace layout in keyboard json - keyboard = keyboard.replace('"LAYOUT_JSON_HERE"', layout) + keyboard = { + 'keyboard_name': kle.name, + 'url': '', + 'maintainer': 'qmk', + 'width': kle.columns, + 'height': kle.rows, + 'layouts': { + 'LAYOUT': { + 'layout': kle2qmk(kle) + } + }, + } + # Write our info.json - file = open(out_path / "info.json", "w") - file.write(keyboard) - file.close() + keyboard = json.dumps(keyboard, indent=4, separators=(', ', ': '), sort_keys=False, cls=InfoJSONEncoder) + info_json_file = out_path / 'info.json' + + info_json_file.write_text(keyboard) cli.log.info('Wrote out {fg_cyan}%s/info.json', out_path) diff --git a/lib/python/qmk/cli/lint.py b/lib/python/qmk/cli/lint.py index 74467021e0..a164dba632 100644 --- a/lib/python/qmk/cli/lint.py +++ b/lib/python/qmk/cli/lint.py @@ -4,12 +4,13 @@ from milc import cli from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json +from qmk.keyboard import keyboard_completer from qmk.keymap import locate_keymap from qmk.path import is_keyboard, keyboard @cli.argument('--strict', action='store_true', help='Treat warnings as errors.') -@cli.argument('-kb', '--keyboard', help='The keyboard to check.') +@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='The keyboard to check.') @cli.argument('-km', '--keymap', help='The keymap to check.') @cli.subcommand('Check keyboard and keymap for common mistakes.') @automagic_keyboard diff --git a/lib/python/qmk/cli/list/keymaps.py b/lib/python/qmk/cli/list/keymaps.py index 49bc84b2ce..d79ab75b58 100644 --- a/lib/python/qmk/cli/list/keymaps.py +++ b/lib/python/qmk/cli/list/keymaps.py @@ -4,18 +4,14 @@ from milc import cli import qmk.keymap from qmk.decorators import automagic_keyboard -from qmk.path import is_keyboard +from qmk.keyboard import keyboard_completer, keyboard_folder -@cli.argument("-kb", "--keyboard", help="Specify keyboard name. Example: 1upkeyboards/1up60hse") +@cli.argument("-kb", "--keyboard", type=keyboard_folder, completer=keyboard_completer, help="Specify keyboard name. Example: 1upkeyboards/1up60hse") @cli.subcommand("List the keymaps for a specific keyboard") @automagic_keyboard def list_keymaps(cli): """List the keymaps for a specific keyboard """ - if not is_keyboard(cli.config.list_keymaps.keyboard): - cli.log.error('Keyboard %s does not exist!', cli.config.list_keymaps.keyboard) - return False - for name in qmk.keymap.list_keymaps(cli.config.list_keymaps.keyboard): print(name) diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py new file mode 100755 index 0000000000..46594c0997 --- /dev/null +++ b/lib/python/qmk/cli/multibuild.py @@ -0,0 +1,79 @@ +"""Compile all keyboards. + +This will compile everything in parallel, for testing purposes. +""" +import re +from pathlib import Path +from subprocess import DEVNULL + +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from qmk.commands import _find_make +import qmk.keyboard + + +def _make_rules_mk_filter(key, value): + def _rules_mk_filter(keyboard_name): + rules_mk = qmk.keyboard.rules_mk(keyboard_name) + return True if key in rules_mk and rules_mk[key].lower() == str(value).lower() else False + + return _rules_mk_filter + + +def _is_split(keyboard_name): + rules_mk = qmk.keyboard.rules_mk(keyboard_name) + return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False + + +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") +@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") +@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True) +def multibuild(cli): + """Compile QMK Firmware against all keyboards. + """ + + make_cmd = _find_make() + if cli.args.clean: + cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL) + + builddir = Path(QMK_FIRMWARE) / '.build' + makefile = builddir / 'parallel_kb_builds.mk' + + keyboard_list = qmk.keyboard.list_keyboards() + + filter_re = re.compile(r'^(?P<key>[A-Z0-9_]+)\s*=\s*(?P<value>[^#]+)$') + for filter_txt in cli.args.filter: + f = filter_re.match(filter_txt) + if f is not None: + keyboard_list = filter(_make_rules_mk_filter(f.group('key'), f.group('value')), keyboard_list) + + keyboard_list = list(sorted(keyboard_list)) + + if len(keyboard_list) == 0: + return + + builddir.mkdir(parents=True, exist_ok=True) + with open(makefile, "w") as f: + for keyboard_name in keyboard_list: + keyboard_safe = keyboard_name.replace('/', '_') + # yapf: disable + f.write( + f"""\ +all: {keyboard_safe}_binary +{keyboard_safe}_binary: + @rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true + +@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="default" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false \\ + >>"{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" 2>&1 \\ + || cp "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" + @{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:default" ; }} \\ + || {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:default" ; }} \\ + || printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:default" + @rm -f "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" || true + +"""# noqa + ) + # yapf: enable + + cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile, 'all'], capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/new/__init__.py b/lib/python/qmk/cli/new/__init__.py index c6a26939b8..fe5d6fe483 100644 --- a/lib/python/qmk/cli/new/__init__.py +++ b/lib/python/qmk/cli/new/__init__.py @@ -1 +1,2 @@ +from . import keyboard from . import keymap diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py new file mode 100644 index 0000000000..ae4445ca48 --- /dev/null +++ b/lib/python/qmk/cli/new/keyboard.py @@ -0,0 +1,11 @@ +"""This script automates the creation of keyboards. +""" +from milc import cli + + +@cli.subcommand('Creates a new keyboard') +def new_keyboard(cli): + """Creates a new keyboard + """ + # TODO: replace this bodge to the existing script + cli.run(['util/new_keyboard.sh'], stdin=None, capture_output=False) diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 52c564997b..60cb743cb6 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -5,10 +5,11 @@ from pathlib import Path import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.keyboard import keyboard_completer, keyboard_folder from milc import cli -@cli.argument('-kb', '--keyboard', help='Specify keyboard name. Example: 1upkeyboards/1up60hse') +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Specify keyboard name. Example: 1upkeyboards/1up60hse') @cli.argument('-km', '--keymap', help='Specify the name for the new keymap directory') @cli.subcommand('Creates a new keymap for the keyboard of your choosing') @automagic_keyboard diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py index 1464443804..abe5f6de19 100755 --- a/lib/python/qmk/cli/pyformat.py +++ b/lib/python/qmk/cli/pyformat.py @@ -1,17 +1,26 @@ """Format python code according to QMK's style. """ -from milc import cli +from subprocess import CalledProcessError, DEVNULL -import subprocess +from milc import cli +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) def pyformat(cli): """Format python code according to QMK's style. """ + edit = '--diff' if cli.args.dry_run else '--in-place' + yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] try: - subprocess.run(['yapf', '-vv', '-ri', 'bin/qmk', 'lib/python'], check=True) - cli.log.info('Successfully formatted the python code in `bin/qmk` and `lib/python`.') + cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) + cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') + return True + + except CalledProcessError: + if cli.args.dry_run: + cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') + else: + cli.log.error('Error formatting python code!') - except subprocess.CalledProcessError: - cli.log.error('Error formatting python code!') + return False diff --git a/lib/python/qmk/cli/pytest.py b/lib/python/qmk/cli/pytest.py index 5417a9cb34..bdb336b9a7 100644 --- a/lib/python/qmk/cli/pytest.py +++ b/lib/python/qmk/cli/pytest.py @@ -2,7 +2,7 @@ QMK script to run unit and integration tests against our python code. """ -import subprocess +from subprocess import DEVNULL from milc import cli @@ -11,6 +11,7 @@ from milc import cli def pytest(cli): """Run several linting/testing commands. """ - flake8 = subprocess.run(['flake8', 'lib/python', 'bin/qmk']) - nose2 = subprocess.run(['nose2', '-v']) + nose2 = cli.run(['nose2', '-v'], capture_output=False, stdin=DEVNULL) + flake8 = cli.run(['flake8', 'lib/python', 'bin/qmk'], capture_output=False, stdin=DEVNULL) + return flake8.returncode | nose2.returncode |