diff options
Diffstat (limited to 'lib/python/qmk/cli')
26 files changed, 1009 insertions, 301 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index de71a5d1e7..b22f1c0d2d 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -40,7 +40,10 @@ subcommands = [ 'qmk.cli.doctor', 'qmk.cli.fileformat', 'qmk.cli.flash', + 'qmk.cli.format.c', 'qmk.cli.format.json', + 'qmk.cli.format.python', + 'qmk.cli.format.text', 'qmk.cli.generate.api', 'qmk.cli.generate.config_h', 'qmk.cli.generate.dfu_header', @@ -50,6 +53,7 @@ subcommands = [ 'qmk.cli.generate.layouts', 'qmk.cli.generate.rgb_breathe_table', 'qmk.cli.generate.rules_mk', + 'qmk.cli.generate.version_h', 'qmk.cli.hello', 'qmk.cli.info', 'qmk.cli.json2c', diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py index efeb459676..9d0ecaeba3 100644..100755 --- a/lib/python/qmk/cli/cformat.py +++ b/lib/python/qmk/cli/cformat.py @@ -1,137 +1,28 @@ -"""Format C code according to QMK's style. +"""Point people to the new command name. """ -from os import path -from shutil import which -from subprocess import CalledProcessError, DEVNULL, Popen, PIPE +import sys +from pathlib import Path -from argcomplete.completers import FilesCompleter from milc import cli -from qmk.path import normpath -from qmk.c_parse import c_source_files - -c_file_suffixes = ('c', 'h', 'cpp') -core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms') -ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios') - - -def find_clang_format(): - """Returns the path to clang-format. - """ - for clang_version in range(20, 6, -1): - binary = f'clang-format-{clang_version}' - - if which(binary): - return binary - - return 'clang-format' - - -def find_diffs(files): - """Run clang-format and diff it against a file. - """ - found_diffs = False - - for file in files: - cli.log.debug('Checking for changes in %s', file) - clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True) - diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True) - - if diff.returncode != 0: - print(diff.stdout) - found_diffs = True - - return found_diffs - - -def cformat_run(files): - """Spawn clang-format subprocess with proper arguments - """ - # Determine which version of clang-format to use - clang_format = [find_clang_format(), '-i'] - - try: - cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL) - cli.log.info('Successfully formatted the C code.') - return True - - except CalledProcessError as e: - cli.log.error('Error formatting C code!') - cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode) - cli.log.debug('STDOUT:') - cli.log.debug(e.stdout) - cli.log.debug('STDERR:') - cli.log.debug(e.stderr) - return False - - -def filter_files(files, core_only=False): - """Yield only files to be formatted and skip the rest - """ - if core_only: - # Filter non-core files - for index, file in enumerate(files): - # The following statement checks each file to see if the file path is - # - in the core directories - # - not in the ignored directories - if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): - files[index] = None - cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) - - for file in files: - if file and file.name.split('.')[-1] in c_file_suffixes: - yield file - else: - cli.log.debug('Skipping file %s', file) - @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') @cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') @cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') -@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') -@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) +@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.') +@cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True) def cformat(cli): - """Format C code according to QMK's style. + """Pointer to the new command name: qmk format-c. """ - # Find the list of files to format - if cli.args.files: - files = list(filter_files(cli.args.files, cli.args.core_only)) - - if not files: - cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) - exit(0) - - if cli.args.all_files: - cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) - - elif cli.args.all_files: - all_files = c_source_files(core_dirs) - files = list(filter_files(all_files, True)) - - else: - git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs] - git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) - - if git_diff.returncode != 0: - cli.log.error("Error running %s", git_diff_cmd) - print(git_diff.stderr) - return git_diff.returncode - - files = [] - - for file in git_diff.stdout.strip().split('\n'): - if not any([file.startswith(ignore) for ignore in ignored]): - if path.exists(file) and file.split('.')[-1] in c_file_suffixes: - files.append(file) + cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('cformat')] = 'format-c' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') - # Sanity check - if not files: - cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files') - return False + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) - # Run clang-format on the files we've found - if cli.args.dry_run: - return not find_diffs(files) - else: - return cformat_run(files) + return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7a45e77214..acbd778649 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,7 @@ from qmk.keymap import keymap_completer @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') diff --git a/lib/python/qmk/cli/console.py b/lib/python/qmk/cli/console.py index 45ff0c8bee..3c508160e3 100644 --- a/lib/python/qmk/cli/console.py +++ b/lib/python/qmk/cli/console.py @@ -48,10 +48,11 @@ KNOWN_BOOTLOADERS = { ('239A', '000C'): 'caterina: Adafruit Feather 32U4', ('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v', ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v', - ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v', ('2A03', '0036'): 'caterina: Arduino Leonardo', ('2A03', '0037'): 'caterina: Arduino Micro', - ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode' + ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode', + ('03EB', '2067'): 'qmk-hid: HID Bootloader', + ('03EB', '2045'): 'lufa-ms: LUFA Mass Storage Bootloader' } diff --git a/lib/python/qmk/cli/doctor/__init__.py b/lib/python/qmk/cli/doctor/__init__.py new file mode 100755 index 0000000000..272e042023 --- /dev/null +++ b/lib/python/qmk/cli/doctor/__init__.py @@ -0,0 +1,5 @@ +"""QMK Doctor + +Check out the user's QMK environment and make sure it's ready to compile. +""" +from .main import doctor diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py new file mode 100644 index 0000000000..0807f41518 --- /dev/null +++ b/lib/python/qmk/cli/doctor/check.py @@ -0,0 +1,164 @@ +"""Check for specific programs. +""" +from enum import Enum +import re +import shutil +from subprocess import DEVNULL + +from milc import cli +from qmk import submodules +from qmk.constants import QMK_FIRMWARE + + +class CheckStatus(Enum): + OK = 1 + WARNING = 2 + ERROR = 3 + + +ESSENTIAL_BINARIES = { + 'dfu-programmer': {}, + 'avrdude': {}, + 'dfu-util': {}, + 'avr-gcc': { + 'version_arg': '-dumpversion' + }, + 'arm-none-eabi-gcc': { + 'version_arg': '-dumpversion' + }, + 'bin/qmk': {}, +} + + +def _parse_gcc_version(version): + m = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", version) + + return { + 'major': int(m.group(1)), + 'minor': int(m.group(2)) if m.group(2) else 0, + 'patch': int(m.group(3)) if m.group(3) else 0, + } + + +def _check_arm_gcc_version(): + """Returns True if the arm-none-eabi-gcc version is not known to cause problems. + """ + if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']: + version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip() + cli.log.info('Found arm-none-eabi-gcc version %s', version_number) + + return CheckStatus.OK # Right now all known arm versions are ok + + +def _check_avr_gcc_version(): + """Returns True if the avr-gcc version is not known to cause problems. + """ + rc = CheckStatus.ERROR + if 'output' in ESSENTIAL_BINARIES['avr-gcc']: + version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip() + + cli.log.info('Found avr-gcc version %s', version_number) + rc = CheckStatus.OK + + parsed_version = _parse_gcc_version(version_number) + if parsed_version['major'] > 8: + cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.') + rc = CheckStatus.WARNING + + return rc + + +def _check_avrdude_version(): + if 'output' in ESSENTIAL_BINARIES['avrdude']: + last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2] + version_number = last_line.split()[2][:-1] + cli.log.info('Found avrdude version %s', version_number) + + return CheckStatus.OK + + +def _check_dfu_util_version(): + if 'output' in ESSENTIAL_BINARIES['dfu-util']: + first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0] + version_number = first_line.split()[1] + cli.log.info('Found dfu-util version %s', version_number) + + return CheckStatus.OK + + +def _check_dfu_programmer_version(): + if 'output' in ESSENTIAL_BINARIES['dfu-programmer']: + first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0] + version_number = first_line.split()[1] + cli.log.info('Found dfu-programmer version %s', version_number) + + return CheckStatus.OK + + +def check_binaries(): + """Iterates through ESSENTIAL_BINARIES and tests them. + """ + ok = True + + for binary in sorted(ESSENTIAL_BINARIES): + if not is_executable(binary): + ok = False + + return ok + + +def check_binary_versions(): + """Check the versions of ESSENTIAL_BINARIES + """ + versions = [] + for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version): + versions.append(check()) + return versions + + +def check_submodules(): + """Iterates through all submodules to make sure they're cloned and up to date. + """ + for submodule in submodules.status().values(): + if submodule['status'] is None: + cli.log.error('Submodule %s has not yet been cloned!', submodule['name']) + return CheckStatus.ERROR + elif not submodule['status']: + cli.log.warning('Submodule %s is not up to date!', submodule['name']) + return CheckStatus.WARNING + + return CheckStatus.OK + + +def is_executable(command): + """Returns True if command exists and can be executed. + """ + # Make sure the command is in the path. + res = shutil.which(command) + if res is None: + cli.log.error("{fg_red}Can't find %s in your path.", command) + return False + + # Make sure the command can be executed + version_arg = ESSENTIAL_BINARIES[command].get('version_arg', '--version') + check = cli.run([command, version_arg], combined_output=True, stdin=DEVNULL, timeout=5) + + ESSENTIAL_BINARIES[command]['output'] = check.stdout + + if check.returncode in [0, 1]: # Older versions of dfu-programmer exit 1 + cli.log.debug('Found {fg_cyan}%s', command) + return True + + cli.log.error("{fg_red}Can't run `%s %s`", command, version_arg) + return False + + +def check_git_repo(): + """Checks that the .git directory exists inside QMK_HOME. + + This is a decent enough indicator that the qmk_firmware directory is a + proper Git repository, rather than a .zip download from GitHub. + """ + dot_git = QMK_FIRMWARE / '.git' + + return CheckStatus.OK if dot_git.exists() else CheckStatus.WARNING diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py new file mode 100644 index 0000000000..6ce00f6ef1 --- /dev/null +++ b/lib/python/qmk/cli/doctor/linux.py @@ -0,0 +1,172 @@ +"""OS-specific functions for: Linux +""" +import platform +import shutil +from pathlib import Path + +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from .check import CheckStatus + + +def _udev_rule(vid, pid=None, *args): + """ Helper function that return udev rules + """ + rule = "" + if pid: + rule = 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", ATTRS{idProduct}=="%s", TAG+="uaccess"' % ( + vid, + pid, + ) + else: + rule = 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", TAG+="uaccess"' % vid + if args: + rule = ', '.join([rule, *args]) + return rule + + +def _deprecated_udev_rule(vid, pid=None): + """ Helper function that return udev rules + + Note: these are no longer the recommended rules, this is just used to check for them + """ + if pid: + return 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", ATTRS{idProduct}=="%s", MODE:="0666"' % (vid, pid) + else: + return 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", MODE:="0666"' % vid + + +def check_udev_rules(): + """Make sure the udev rules look good. + """ + rc = CheckStatus.OK + udev_dirs = [ + Path("/usr/lib/udev/rules.d/"), + Path("/usr/local/lib/udev/rules.d/"), + Path("/run/udev/rules.d/"), + Path("/etc/udev/rules.d/"), + ] + desired_rules = { + 'atmel-dfu': { + _udev_rule("03eb", "2fef"), # ATmega16U2 + _udev_rule("03eb", "2ff0"), # ATmega32U2 + _udev_rule("03eb", "2ff3"), # ATmega16U4 + _udev_rule("03eb", "2ff4"), # ATmega32U4 + _udev_rule("03eb", "2ff9"), # AT90USB64 + _udev_rule("03eb", "2ffa"), # AT90USB162 + _udev_rule("03eb", "2ffb") # AT90USB128 + }, + 'kiibohd': {_udev_rule("1c11", "b007")}, + 'stm32': { + _udev_rule("1eaf", "0003"), # STM32duino + _udev_rule("0483", "df11") # STM32 DFU + }, + 'bootloadhid': {_udev_rule("16c0", "05df")}, + 'usbasploader': {_udev_rule("16c0", "05dc")}, + 'massdrop': {_udev_rule("03eb", "6124", 'ENV{ID_MM_DEVICE_IGNORE}="1"')}, + 'caterina': { + # Spark Fun Electronics + _udev_rule("1b4f", "9203", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 3V3/8MHz + _udev_rule("1b4f", "9205", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 5V/16MHz + _udev_rule("1b4f", "9207", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # LilyPad 3V3/8MHz (and some Pro Micro clones) + # Pololu Electronics + _udev_rule("1ffb", "0101", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # A-Star 32U4 + # Arduino SA + _udev_rule("2341", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo + _udev_rule("2341", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Micro + # Adafruit Industries LLC + _udev_rule("239a", "000c", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Feather 32U4 + _udev_rule("239a", "000d", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 3V3/8MHz + _udev_rule("239a", "000e", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 5V/16MHz + # dog hunter AG + _udev_rule("2a03", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo + _udev_rule("2a03", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"') # Micro + }, + 'hid-bootloader': { + _udev_rule("03eb", "2067"), # QMK HID + _udev_rule("16c0", "0478") # PJRC halfkay + } + } + + # These rules are no longer recommended, only use them to check for their presence. + deprecated_rules = { + 'atmel-dfu': {_deprecated_udev_rule("03eb", "2ff4"), _deprecated_udev_rule("03eb", "2ffb"), _deprecated_udev_rule("03eb", "2ff0")}, + 'kiibohd': {_deprecated_udev_rule("1c11")}, + 'stm32': {_deprecated_udev_rule("1eaf", "0003"), _deprecated_udev_rule("0483", "df11")}, + 'bootloadhid': {_deprecated_udev_rule("16c0", "05df")}, + 'caterina': {'ATTRS{idVendor}=="2a03", ENV{ID_MM_DEVICE_IGNORE}="1"', 'ATTRS{idVendor}=="2341", ENV{ID_MM_DEVICE_IGNORE}="1"'}, + 'tmk': {_deprecated_udev_rule("feed")} + } + + if any(udev_dir.exists() for udev_dir in udev_dirs): + udev_rules = [rule_file for udev_dir in udev_dirs for rule_file in udev_dir.glob('*.rules')] + current_rules = set() + + # Collect all rules from the config files + for rule_file in udev_rules: + for line in rule_file.read_text(encoding='utf-8').split('\n'): + line = line.strip() + if not line.startswith("#") and len(line): + current_rules.add(line) + + # Check if the desired rules are among the currently present rules + for bootloader, rules in desired_rules.items(): + if not rules.issubset(current_rules): + deprecated_rule = deprecated_rules.get(bootloader) + if deprecated_rule and deprecated_rule.issubset(current_rules): + cli.log.warning("{fg_yellow}Found old, deprecated udev rules for '%s' boards. The new rules on https://docs.qmk.fm/#/faq_build?id=linux-udev-rules offer better security with the same functionality.", bootloader) + else: + # For caterina, check if ModemManager is running + if bootloader == "caterina": + if check_modem_manager(): + rc = CheckStatus.WARNING + cli.log.warning("{fg_yellow}Detected ModemManager without the necessary udev rules. Please either disable it or set the appropriate udev rules if you are using a Pro Micro.") + rc = CheckStatus.WARNING + cli.log.warning("{fg_yellow}Missing or outdated udev rules for '%s' boards. Run 'sudo cp %s/util/udev/50-qmk.rules /etc/udev/rules.d/'.", bootloader, QMK_FIRMWARE) + + else: + cli.log.warning("{fg_yellow}Can't find udev rules, skipping udev rule checking...") + cli.log.debug("Checked directories: %s", ', '.join(str(udev_dir) for udev_dir in udev_dirs)) + + return rc + + +def check_systemd(): + """Check if it's a systemd system + """ + return bool(shutil.which("systemctl")) + + +def check_modem_manager(): + """Returns True if ModemManager is running. + + """ + if check_systemd(): + mm_check = cli.run(["systemctl", "--quiet", "is-active", "ModemManager.service"], timeout=10) + if mm_check.returncode == 0: + return True + else: + """(TODO): Add check for non-systemd systems + """ + return False + + +def os_test_linux(): + """Run the Linux specific tests. + """ + # Don't bother with udev on WSL, for now + if 'microsoft' in platform.uname().release.lower(): + cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.") + + # https://github.com/microsoft/WSL/issues/4197 + if QMK_FIRMWARE.as_posix().startswith("/mnt"): + cli.log.warning("I/O performance on /mnt may be extremely slow.") + return CheckStatus.WARNING + + return CheckStatus.OK + else: + cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") + from .linux import check_udev_rules + + return check_udev_rules() diff --git a/lib/python/qmk/cli/doctor/macos.py b/lib/python/qmk/cli/doctor/macos.py new file mode 100644 index 0000000000..00fb272858 --- /dev/null +++ b/lib/python/qmk/cli/doctor/macos.py @@ -0,0 +1,13 @@ +import platform + +from milc import cli + +from .check import CheckStatus + + +def os_test_macos(): + """Run the Mac specific tests. + """ + cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) + + return CheckStatus.OK diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor/main.py index 327bc9cb30..6a31ccdfdd 100755 --- a/lib/python/qmk/cli/doctor.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -7,9 +7,11 @@ from subprocess import DEVNULL from milc import cli from milc.questions import yesno + from qmk import submodules -from qmk.constants import QMK_FIRMWARE -from qmk.os_helpers import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules +from qmk.commands import git_check_repo, git_get_branch, git_is_dirty, git_get_remotes, git_check_deviation, in_virtualenv def os_tests(): @@ -18,51 +20,48 @@ def os_tests(): platform_id = platform.platform().lower() if 'darwin' in platform_id or 'macos' in platform_id: + from .macos import os_test_macos return os_test_macos() elif 'linux' in platform_id: + from .linux import os_test_linux return os_test_linux() elif 'windows' in platform_id: + from .windows import os_test_windows return os_test_windows() else: cli.log.warning('Unsupported OS detected: %s', platform_id) return CheckStatus.WARNING -def os_test_linux(): - """Run the Linux specific tests. +def git_tests(): + """Run Git-related checks """ - # Don't bother with udev on WSL, for now - if 'microsoft' in platform.uname().release.lower(): - cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.") - - # https://github.com/microsoft/WSL/issues/4197 - if QMK_FIRMWARE.as_posix().startswith("/mnt"): - cli.log.warning("I/O performance on /mnt may be extremely slow.") - return CheckStatus.WARNING + status = CheckStatus.OK - return CheckStatus.OK + # Make sure our QMK home is a Git repo + git_ok = git_check_repo() + if not git_ok: + cli.log.warning("{fg_yellow}QMK home does not appear to be a Git repository! (no .git folder)") + status = CheckStatus.WARNING else: - cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") - from qmk.os_helpers.linux import check_udev_rules - - return check_udev_rules() - - -def os_test_macos(): - """Run the Mac specific tests. - """ - cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) - - return CheckStatus.OK - - -def os_test_windows(): - """Run the Windows specific tests. - """ - win32_ver = platform.win32_ver() - cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1]) - - return CheckStatus.OK + git_branch = git_get_branch() + if git_branch: + cli.log.info('Git branch: %s', git_branch) + git_dirty = git_is_dirty() + if git_dirty: + cli.log.warning('{fg_yellow}Git has unstashed/uncommitted changes.') + status = CheckStatus.WARNING + git_remotes = git_get_remotes() + if 'upstream' not in git_remotes.keys() or QMK_FIRMWARE_UPSTREAM not in git_remotes['upstream'].get('url', ''): + cli.log.warning('{fg_yellow}The official repository does not seem to be configured as git remote "upstream".') + status = CheckStatus.WARNING + else: + git_deviation = git_check_deviation(git_branch) + if git_branch in ['master', 'develop'] and git_deviation: + cli.log.warning('{fg_yellow}The local "%s" branch contains commits not found in the upstream branch.', git_branch) + status = CheckStatus.WARNING + + return status @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') @@ -82,12 +81,11 @@ def doctor(cli): status = os_tests() - # Make sure our QMK home is a Git repo - git_ok = check_git_repo() + status = git_tests() - if git_ok == CheckStatus.WARNING: - cli.log.warning("QMK home does not appear to be a Git repository! (no .git folder)") - status = CheckStatus.WARNING + venv = in_virtualenv() + if venv: + cli.log.info('CLI installed in virtualenv.') # Make sure the basic CLI tools we need are available and can be executed. bin_ok = check_binaries() diff --git a/lib/python/qmk/cli/doctor/windows.py b/lib/python/qmk/cli/doctor/windows.py new file mode 100644 index 0000000000..381ab36fde --- /dev/null +++ b/lib/python/qmk/cli/doctor/windows.py @@ -0,0 +1,14 @@ +import platform + +from milc import cli + +from .check import CheckStatus + + +def os_test_windows(): + """Run the Windows specific tests. + """ + win32_ver = platform.win32_ver() + cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1]) + + return CheckStatus.OK diff --git a/lib/python/qmk/cli/fileformat.py b/lib/python/qmk/cli/fileformat.py index 112d8d59da..cee4ba1acd 100644..100755 --- a/lib/python/qmk/cli/fileformat.py +++ b/lib/python/qmk/cli/fileformat.py @@ -1,13 +1,23 @@ -"""Format files according to QMK's style. +"""Point people to the new command name. """ -from milc import cli +import sys +from pathlib import Path -import subprocess +from milc import cli -@cli.subcommand("Format files according to QMK's style.", hidden=True) +@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True) def fileformat(cli): - """Run several general formatting commands. + """Pointer to the new command name: qmk format-text. """ - dos2unix = subprocess.run(['bash', '-c', 'git ls-files -z | xargs -0 dos2unix'], stdout=subprocess.DEVNULL) - return dos2unix.returncode + cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('fileformat')] = 'format-text' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') + + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) + + return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 1b2932a5b2..c2d9e09c69 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -38,7 +38,7 @@ def print_bootloader_help(): @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('QMK Flash.') diff --git a/lib/python/qmk/cli/format/c.py b/lib/python/qmk/cli/format/c.py new file mode 100644 index 0000000000..0160e6036f --- /dev/null +++ b/lib/python/qmk/cli/format/c.py @@ -0,0 +1,137 @@ +"""Format C code according to QMK's style. +""" +from os import path +from shutil import which +from subprocess import CalledProcessError, DEVNULL, Popen, PIPE + +from argcomplete.completers import FilesCompleter +from milc import cli + +from qmk.path import normpath +from qmk.c_parse import c_source_files + +c_file_suffixes = ('c', 'h', 'cpp') +core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms') +ignored = ('tmk_core/protocol/usb_hid', 'platforms/chibios/boards') + + +def find_clang_format(): + """Returns the path to clang-format. + """ + for clang_version in range(20, 6, -1): + binary = f'clang-format-{clang_version}' + + if which(binary): + return binary + + return 'clang-format' + + +def find_diffs(files): + """Run clang-format and diff it against a file. + """ + found_diffs = False + + for file in files: + cli.log.debug('Checking for changes in %s', file) + clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True) + diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True) + + if diff.returncode != 0: + print(diff.stdout) + found_diffs = True + + return found_diffs + + +def cformat_run(files): + """Spawn clang-format subprocess with proper arguments + """ + # Determine which version of clang-format to use + clang_format = [find_clang_format(), '-i'] + + try: + cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL) + cli.log.info('Successfully formatted the C code.') + return True + + except CalledProcessError as e: + cli.log.error('Error formatting C code!') + cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode) + cli.log.debug('STDOUT:') + cli.log.debug(e.stdout) + cli.log.debug('STDERR:') + cli.log.debug(e.stderr) + return False + + +def filter_files(files, core_only=False): + """Yield only files to be formatted and skip the rest + """ + if core_only: + # Filter non-core files + for index, file in enumerate(files): + # The following statement checks each file to see if the file path is + # - in the core directories + # - not in the ignored directories + if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): + files[index] = None + cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) + + for file in files: + if file and file.name.split('.')[-1] in c_file_suffixes: + yield file + else: + cli.log.debug('Skipping file %s', file) + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") +@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') +@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') +@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') +@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') +@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) +def format_c(cli): + """Format C code according to QMK's style. + """ + # Find the list of files to format + if cli.args.files: + files = list(filter_files(cli.args.files, cli.args.core_only)) + + if not files: + cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) + exit(0) + + if cli.args.all_files: + cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) + + elif cli.args.all_files: + all_files = c_source_files(core_dirs) + files = list(filter_files(all_files, True)) + + else: + git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs] + git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) + + if git_diff.returncode != 0: + cli.log.error("Error running %s", git_diff_cmd) + print(git_diff.stderr) + return git_diff.returncode + + files = [] + + for file in git_diff.stdout.strip().split('\n'): + if not any([file.startswith(ignore) for ignore in ignored]): + if path.exists(file) and file.split('.')[-1] in c_file_suffixes: + files.append(file) + + # Sanity check + if not files: + cli.log.error('No changed files detected. Use "qmk format-c -a" to format all core files') + return False + + # Run clang-format on the files we've found + if cli.args.dry_run: + return not find_diffs(files) + else: + return cformat_run(files) diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 1358c70e7a..19d504491f 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -8,7 +8,7 @@ from jsonschema import ValidationError from milc import cli from qmk.info import info_json -from qmk.json_schema import json_load, keyboard_validate +from qmk.json_schema import json_load, validate from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder from qmk.path import normpath @@ -23,14 +23,13 @@ def format_json(cli): if cli.args.format == 'auto': try: - keyboard_validate(json_file) + validate(json_file, 'qmk.keyboard.v1') json_encoder = InfoJSONEncoder except ValidationError as e: cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) cli.log.info('Treating %s as a keymap file.', cli.args.json_file) json_encoder = KeymapJSONEncoder - elif cli.args.format == 'keyboard': json_encoder = InfoJSONEncoder elif cli.args.format == 'keymap': diff --git a/lib/python/qmk/cli/format/python.py b/lib/python/qmk/cli/format/python.py new file mode 100755 index 0000000000..00612f97ec --- /dev/null +++ b/lib/python/qmk/cli/format/python.py @@ -0,0 +1,26 @@ +"""Format python code according to QMK's style. +""" +from subprocess import CalledProcessError, DEVNULL + +from milc import cli + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") +@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) +def format_python(cli): + """Format python code according to QMK's style. + """ + edit = '--diff' if cli.args.dry_run else '--in-place' + yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] + try: + cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) + cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') + return True + + except CalledProcessError: + if cli.args.dry_run: + cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') + else: + cli.log.error('Error formatting python code!') + + return False diff --git a/lib/python/qmk/cli/format/text.py b/lib/python/qmk/cli/format/text.py new file mode 100644 index 0000000000..e7e07b7297 --- /dev/null +++ b/lib/python/qmk/cli/format/text.py @@ -0,0 +1,27 @@ +"""Ensure text files have the proper line endings. +""" +from subprocess import CalledProcessError + +from milc import cli + + +@cli.subcommand("Ensure text files have the proper line endings.", hidden=True) +def format_text(cli): + """Ensure text files have the proper line endings. + """ + try: + file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True) + except CalledProcessError as e: + cli.log.error('Could not get file list: %s', e) + exit(1) + except Exception as e: + cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e) + cli.log.exception(e) + exit(1) + + dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout) + + if dos2unix.returncode != 0: + print(dos2unix.stderr) + + return dos2unix.returncode diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index 54cd5b96a8..ca7e14fe6b 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -5,14 +5,14 @@ from pathlib import Path from dotty_dict import dotty from milc import cli -from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json -from qmk.json_schema import json_load +from qmk.json_schema import json_load, validate from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.path import is_keyboard, normpath +from qmk.keymap import locate_keymap +from qmk.path import normpath -def direct_pins(direct_pins): +def direct_pins(direct_pins, postfix): """Return the config.h lines that set the direct pins. """ rows = [] @@ -24,81 +24,60 @@ def direct_pins(direct_pins): col_count = len(direct_pins[0]) row_count = len(direct_pins) - return """ -#ifndef MATRIX_COLS -# define MATRIX_COLS %s -#endif // MATRIX_COLS + return f""" +#ifndef MATRIX_COLS{postfix} +# define MATRIX_COLS{postfix} {col_count} +#endif // MATRIX_COLS{postfix} -#ifndef MATRIX_ROWS -# define MATRIX_ROWS %s -#endif // MATRIX_ROWS +#ifndef MATRIX_ROWS{postfix} +# define MATRIX_ROWS{postfix} {row_count} +#endif // MATRIX_ROWS{postfix} -#ifndef DIRECT_PINS -# define DIRECT_PINS {%s} -#endif // DIRECT_PINS -""" % (col_count, row_count, ','.join(rows)) +#ifndef DIRECT_PINS{postfix} +# define DIRECT_PINS{postfix} {{ {", ".join(rows)} }} +#endif // DIRECT_PINS{postfix} +""" -def pin_array(define, pins): +def pin_array(define, pins, postfix): """Return the config.h lines that set a pin array. """ pin_num = len(pins) pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins])) return f""" -#ifndef {define}S -# define {define}S {pin_num} -#endif // {define}S +#ifndef {define}S{postfix} +# define {define}S{postfix} {pin_num} +#endif // {define}S{postfix} -#ifndef {define}_PINS -# define {define}_PINS {{ {pin_array} }} -#endif // {define}_PINS +#ifndef {define}_PINS{postfix} +# define {define}_PINS{postfix} {{ {pin_array} }} +#endif // {define}_PINS{postfix} """ -def matrix_pins(matrix_pins): +def matrix_pins(matrix_pins, postfix=''): """Add the matrix config to the config.h. """ pins = [] if 'direct' in matrix_pins: - pins.append(direct_pins(matrix_pins['direct'])) + pins.append(direct_pins(matrix_pins['direct'], postfix)) if 'cols' in matrix_pins: - pins.append(pin_array('MATRIX_COL', matrix_pins['cols'])) + pins.append(pin_array('MATRIX_COL', matrix_pins['cols'], postfix)) if 'rows' in matrix_pins: - pins.append(pin_array('MATRIX_ROW', matrix_pins['rows'])) + pins.append(pin_array('MATRIX_ROW', matrix_pins['rows'], postfix)) return '\n'.join(pins) -@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') -@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') -@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) -@automagic_keyboard -@automagic_keymap -def generate_config_h(cli): - """Generates the info_config.h file. +def generate_config_items(kb_info_json, config_h_lines): + """Iterate through the info_config map to generate basic config values. """ - # Determine our keyboard(s) - if not cli.config.generate_config_h.keyboard: - cli.log.error('Missing parameter: --keyboard') - cli.subcommands['info'].print_help() - return False - - if not is_keyboard(cli.config.generate_config_h.keyboard): - cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard) - return False - - # Build the info_config.h file. - kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard)) info_config_map = json_load(Path('data/mappings/info_config.json')) - config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once'] - - # Iterate through the info_config map to generate basic things for config_key, info_dict in info_config_map.items(): info_key = info_dict['info_key'] key_type = info_dict.get('value_type', 'str') @@ -135,9 +114,75 @@ def generate_config_h(cli): config_h_lines.append(f'# define {config_key} {config_value}') config_h_lines.append(f'#endif // {config_key}') + +def generate_split_config(kb_info_json, config_h_lines): + """Generate the config.h lines for split boards.""" + if 'primary' in kb_info_json['split']: + if kb_info_json['split']['primary'] in ('left', 'right'): + config_h_lines.append('') + config_h_lines.append('#ifndef MASTER_LEFT') + config_h_lines.append('# ifndef MASTER_RIGHT') + if kb_info_json['split']['primary'] == 'left': + config_h_lines.append('# define MASTER_LEFT') + elif kb_info_json['split']['primary'] == 'right': + config_h_lines.append('# define MASTER_RIGHT') + config_h_lines.append('# endif // MASTER_RIGHT') + config_h_lines.append('#endif // MASTER_LEFT') + elif kb_info_json['split']['primary'] == 'pin': + config_h_lines.append('') + config_h_lines.append('#ifndef SPLIT_HAND_PIN') + config_h_lines.append('# define SPLIT_HAND_PIN') + config_h_lines.append('#endif // SPLIT_HAND_PIN') + elif kb_info_json['split']['primary'] == 'matrix_grid': + config_h_lines.append('') + config_h_lines.append('#ifndef SPLIT_HAND_MATRIX_GRID') + config_h_lines.append('# define SPLIT_HAND_MATRIX_GRID {%s}' % (','.join(kb_info_json["split"]["matrix_grid"],))) + config_h_lines.append('#endif // SPLIT_HAND_MATRIX_GRID') + elif kb_info_json['split']['primary'] == 'eeprom': + config_h_lines.append('') + config_h_lines.append('#ifndef EE_HANDS') + config_h_lines.append('# define EE_HANDS') + config_h_lines.append('#endif // EE_HANDS') + + if 'protocol' in kb_info_json['split'].get('transport', {}): + if kb_info_json['split']['transport']['protocol'] == 'i2c': + config_h_lines.append('') + config_h_lines.append('#ifndef USE_I2C') + config_h_lines.append('# define USE_I2C') + config_h_lines.append('#endif // USE_I2C') + + if 'right' in kb_info_json['split'].get('matrix_pins', {}): + config_h_lines.append(matrix_pins(kb_info_json['split']['matrix_pins']['right'], '_RIGHT')) + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate config.h for.') +@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate config.h for.') +@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) +def generate_config_h(cli): + """Generates the info_config.h file. + """ + # Determine our keyboard/keymap + if cli.args.keymap: + km = locate_keymap(cli.args.keyboard, cli.args.keymap) + km_json = json_load(km) + validate(km_json, 'qmk.keymap.v1') + kb_info_json = dotty(km_json.get('config', {})) + else: + kb_info_json = dotty(info_json(cli.args.keyboard)) + + # Build the info_config.h file. + config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once'] + + generate_config_items(kb_info_json, config_h_lines) + if 'matrix_pins' in kb_info_json: config_h_lines.append(matrix_pins(kb_info_json['matrix_pins'])) + if 'split' in kb_info_json: + generate_split_config(kb_info_json, config_h_lines) + # Show the results config_h = '\n'.join(config_h_lines) diff --git a/lib/python/qmk/cli/generate/info_json.py b/lib/python/qmk/cli/generate/info_json.py index 8931b68b6f..284d1a8510 100755 --- a/lib/python/qmk/cli/generate/info_json.py +++ b/lib/python/qmk/cli/generate/info_json.py @@ -4,15 +4,17 @@ Compile an info.json for a particular keyboard and pretty-print it. """ import json -from jsonschema import Draft7Validator, validators +from argcomplete.completers import FilesCompleter +from jsonschema import Draft7Validator, RefResolver, validators from milc import cli +from pathlib import Path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json from qmk.json_encoders import InfoJSONEncoder -from qmk.json_schema import load_jsonschema +from qmk.json_schema import compile_schema_store from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.path import is_keyboard +from qmk.path import is_keyboard, normpath def pruning_validator(validator_class): @@ -34,15 +36,19 @@ def pruning_validator(validator_class): def strip_info_json(kb_info_json): """Remove the API-only properties from the info.json. """ + schema_store = compile_schema_store() pruning_draft_7_validator = pruning_validator(Draft7Validator) - schema = load_jsonschema('keyboard') - validator = pruning_draft_7_validator(schema).validate + schema = schema_store['qmk.keyboard.v1'] + resolver = RefResolver.from_schema(schema_store['qmk.keyboard.v1'], store=schema_store) + validator = pruning_draft_7_validator(schema, resolver=resolver).validate return validator(kb_info_json) @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to show info for.') @cli.argument('-km', '--keymap', help='Show the layers for a JSON keymap too.') +@cli.argument('-o', '--output', arg_only=True, completer=FilesCompleter, help='Write the output the specified file, overwriting if necessary.') +@cli.argument('-ow', '--overwrite', arg_only=True, action='store_true', help='Overwrite the existing info.json. (Overrides the location of --output)') @cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) @automagic_keyboard @automagic_keymap @@ -59,9 +65,29 @@ def generate_info_json(cli): cli.log.error('Invalid keyboard: "%s"', cli.config.generate_info_json.keyboard) return False + if cli.args.overwrite: + output_path = (Path('keyboards') / cli.config.generate_info_json.keyboard / 'info.json').resolve() + + if cli.args.output: + cli.log.warning('Overwriting user supplied --output with %s', output_path) + + cli.args.output = output_path + # Build the info.json file kb_info_json = info_json(cli.config.generate_info_json.keyboard) strip_info_json(kb_info_json) + info_json_text = json.dumps(kb_info_json, indent=4, cls=InfoJSONEncoder) + + if cli.args.output: + # Write to a file + output_path = normpath(cli.args.output) + + if output_path.exists(): + cli.log.warning('Overwriting output file %s', output_path) + + output_path.write_text(info_json_text + '\n') + cli.log.info('Wrote info.json to %s.', output_path) - # Display the results - print(json.dumps(kb_info_json, indent=2, cls=InfoJSONEncoder)) + else: + # Display the results + print(info_json_text) diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py index 22500dbc91..c9d7f549b3 100755 --- a/lib/python/qmk/cli/generate/keyboard_h.py +++ b/lib/python/qmk/cli/generate/keyboard_h.py @@ -2,7 +2,6 @@ """ from milc import cli -from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.path import normpath @@ -29,14 +28,12 @@ def would_populate_layout_h(keyboard): @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.') +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.') @cli.subcommand('Used by the make system to generate keyboard.h from info.json', hidden=True) -@automagic_keyboard -@automagic_keymap def generate_keyboard_h(cli): """Generates the keyboard.h file. """ - has_layout_h = would_populate_layout_h(cli.config.generate_keyboard_h.keyboard) + has_layout_h = would_populate_layout_h(cli.args.keyboard) # Build the layouts.h file. keyboard_h_lines = ['/* This file was generated by `qmk generate-keyboard-h`. Do not edit or copy.' ' */', '', '#pragma once', '#include "quantum.h"'] diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index 41c94e16b5..cdf17dfbcb 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -5,11 +5,11 @@ from pathlib import Path from dotty_dict import dotty from milc import cli -from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json -from qmk.json_schema import json_load +from qmk.json_schema import json_load, validate from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.path import is_keyboard, normpath +from qmk.keymap import locate_keymap +from qmk.path import normpath def process_mapping_rule(kb_info_json, rules_key, info_dict): @@ -39,23 +39,21 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict): @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.argument('-e', '--escape', arg_only=True, action='store_true', help="Escape spaces in quiet mode") -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.') -@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True) -@automagic_keyboard -@automagic_keymap +@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate rules.mk for.') +@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate rules.mk for.') +@cli.subcommand('Used by the make system to generate rules.mk from info.json', hidden=True) def generate_rules_mk(cli): """Generates a rules.mk file from info.json. """ - if not cli.config.generate_rules_mk.keyboard: - cli.log.error('Missing parameter: --keyboard') - cli.subcommands['info'].print_help() - return False - - if not is_keyboard(cli.config.generate_rules_mk.keyboard): - cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard) - return False + # Determine our keyboard/keymap + if cli.args.keymap: + km = locate_keymap(cli.args.keyboard, cli.args.keymap) + km_json = json_load(km) + validate(km_json, 'qmk.keymap.v1') + kb_info_json = dotty(km_json.get('config', {})) + else: + kb_info_json = dotty(info_json(cli.args.keyboard)) - kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard)) info_rules_map = json_load(Path('data/mappings/info_rules.json')) rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', ''] @@ -76,6 +74,17 @@ def generate_rules_mk(cli): enabled = 'yes' if enabled else 'no' rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') + # Set SPLIT_TRANSPORT, if needed + if kb_info_json.get('split', {}).get('transport', {}).get('protocol') == 'custom': + rules_mk_lines.append('SPLIT_TRANSPORT ?= custom') + + # Set CUSTOM_MATRIX, if needed + if kb_info_json.get('matrix_pins', {}).get('custom'): + if kb_info_json.get('matrix_pins', {}).get('custom_lite'): + rules_mk_lines.append('CUSTOM_MATRIX ?= lite') + else: + rules_mk_lines.append('CUSTOM_MATRIX ?= yes') + # Show the results rules_mk = '\n'.join(rules_mk_lines) + '\n' diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py new file mode 100644 index 0000000000..b8e52588c4 --- /dev/null +++ b/lib/python/qmk/cli/generate/version_h.py @@ -0,0 +1,28 @@ +"""Used by the make system to generate version.h for use in code. +""" +from milc import cli + +from qmk.commands import create_version_h +from qmk.path import normpath + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('--skip-git', arg_only=True, action='store_true', help='Skip Git operations') +@cli.argument('--skip-all', arg_only=True, action='store_true', help='Use placeholder values for all defines (implies --skip-git)') +@cli.subcommand('Used by the make system to generate version.h for use in code', hidden=True) +def generate_version_h(cli): + """Generates the version.h file. + """ + if cli.args.skip_all: + cli.args.skip_git = True + + version_h = create_version_h(cli.args.skip_git, cli.args.skip_all) + + if cli.args.output: + cli.args.output.write_text(version_h) + + if not cli.args.quiet: + cli.log.info('Wrote version.h to %s.', cli.args.output) + else: + print(version_h) diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py index 0d08d242cd..3131d4b53f 100755 --- a/lib/python/qmk/cli/info.py +++ b/lib/python/qmk/cli/info.py @@ -24,19 +24,15 @@ def show_keymap(kb_info_json, title_caps=True): keymap_path = locate_keymap(cli.config.info.keyboard, cli.config.info.keymap) if keymap_path and keymap_path.suffix == '.json': - if title_caps: - cli.echo('{fg_blue}Keymap "%s"{fg_reset}:', cli.config.info.keymap) - else: - cli.echo('{fg_blue}keymap_%s{fg_reset}:', cli.config.info.keymap) - keymap_data = json.load(keymap_path.open(encoding='utf-8')) layout_name = keymap_data['layout'] + layout_name = kb_info_json.get('layout_aliases', {}).get(layout_name, layout_name) # Resolve alias names for layer_num, layer in enumerate(keymap_data['layers']): if title_caps: - cli.echo('{fg_cyan}Layer %s{fg_reset}:', layer_num) + cli.echo('{fg_cyan}Keymap %s Layer %s{fg_reset}:', cli.config.info.keymap, layer_num) else: - cli.echo('{fg_cyan}layer_%s{fg_reset}:', layer_num) + cli.echo('{fg_cyan}keymap.%s.layer.%s{fg_reset}:', cli.config.info.keymap, layer_num) print(render_layout(kb_info_json['layouts'][layout_name]['layout'], cli.config.info.ascii, layer)) @@ -45,7 +41,7 @@ def show_layouts(kb_info_json, title_caps=True): """Render the layouts with info.json labels. """ for layout_name, layout_art in render_layouts(kb_info_json, cli.config.info.ascii).items(): - title = layout_name.title() if title_caps else layout_name + title = f'Layout {layout_name.title()}' if title_caps else f'layouts.{layout_name}' cli.echo('{fg_cyan}%s{fg_reset}:', title) print(layout_art) # Avoid passing dirty data to cli.echo() @@ -87,23 +83,12 @@ def print_friendly_output(kb_info_json): cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer']) cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown')) cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys()))) - if 'width' in kb_info_json and 'height' in kb_info_json: - cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height'])) cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown')) cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown')) if 'layout_aliases' in kb_info_json: aliases = [f'{key}={value}' for key, value in kb_info_json['layout_aliases'].items()] cli.echo('{fg_blue}Layout aliases:{fg_reset} %s' % (', '.join(aliases),)) - if cli.config.info.layouts: - show_layouts(kb_info_json, True) - - if cli.config.info.matrix: - show_matrix(kb_info_json, True) - - if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file': - show_keymap(kb_info_json, True) - def print_text_output(kb_info_json): """Print the info.json in a plain text format. @@ -124,6 +109,24 @@ def print_text_output(kb_info_json): show_keymap(kb_info_json, False) +def print_dotted_output(kb_info_json, prefix=''): + """Print the info.json in a plain text format with dot-joined keys. + """ + for key in sorted(kb_info_json): + new_prefix = f'{prefix}.{key}' if prefix else key + + if key in ['parse_errors', 'parse_warnings']: + continue + elif key == 'layouts' and prefix == '': + cli.echo('{fg_blue}layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys()))) + elif isinstance(kb_info_json[key], dict): + print_dotted_output(kb_info_json[key], new_prefix) + elif isinstance(kb_info_json[key], list): + cli.echo('{fg_blue}%s{fg_reset}: %s', new_prefix, ', '.join(map(str, sorted(kb_info_json[key])))) + else: + cli.echo('{fg_blue}%s{fg_reset}: %s', new_prefix, kb_info_json[key]) + + def print_parsed_rules_mk(keyboard_name): rules = rules_mk(keyboard_name) for k in sorted(rules.keys()): @@ -164,10 +167,22 @@ def info(cli): # Output in the requested format if cli.args.format == 'json': print(json.dumps(kb_info_json, cls=InfoJSONEncoder)) + return True elif cli.args.format == 'text': - print_text_output(kb_info_json) + print_dotted_output(kb_info_json) + title_caps = False elif cli.args.format == 'friendly': print_friendly_output(kb_info_json) + title_caps = True else: cli.log.error('Unknown format: %s', cli.args.format) return False + + if cli.config.info.layouts: + show_layouts(kb_info_json, title_caps) + + if cli.config.info.matrix: + show_matrix(kb_info_json, title_caps) + + if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file': + show_keymap(kb_info_json, title_caps) diff --git a/lib/python/qmk/cli/kle2json.py b/lib/python/qmk/cli/kle2json.py index acb75ef4fd..bbfddf4268 100755 --- a/lib/python/qmk/cli/kle2json.py +++ b/lib/python/qmk/cli/kle2json.py @@ -44,8 +44,6 @@ def kle2json(cli): 'keyboard_name': kle.name, 'url': '', 'maintainer': 'qmk', - 'width': kle.columns, - 'height': kle.rows, 'layouts': { 'LAYOUT': { 'layout': kle2qmk(kle) diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py index bdb0b493c8..85ed0fa1e9 100755 --- a/lib/python/qmk/cli/multibuild.py +++ b/lib/python/qmk/cli/multibuild.py @@ -10,7 +10,7 @@ from subprocess import DEVNULL from milc import cli from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make +from qmk.commands import _find_make, get_make_parallel_args import qmk.keyboard import qmk.keymap @@ -28,7 +28,7 @@ def _is_split(keyboard_name): return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") @@ -80,7 +80,7 @@ all: {keyboard_safe}_binary ) # yapf: enable - cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) + cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) # Check for failures failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index ae4445ca48..369d2bd7da 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -1,11 +1,142 @@ -"""This script automates the creation of keyboards. +"""This script automates the creation of new keyboard directories using a starter template. """ +from datetime import date +import fileinput +from pathlib import Path +import re +import shutil + +from qmk.commands import git_get_username +import qmk.path from milc import cli +from milc.questions import choice, question + +KEYBOARD_TYPES = ['avr', 'ps2avrgb'] + + +def keyboard_name(name): + """Callable for argparse validation. + """ + if not validate_keyboard_name(name): + raise ValueError + return name -@cli.subcommand('Creates a new keyboard') +def validate_keyboard_name(name): + """Returns True if the given keyboard name contains only lowercase a-z, 0-9 and underscore characters. + """ + regex = re.compile(r'^[a-z0-9][a-z0-9/_]+$') + return bool(regex.match(name)) + + +@cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name) +@cli.argument('-t', '--type', help='Specify the keyboard type', arg_only=True, choices=KEYBOARD_TYPES) +@cli.argument('-u', '--username', help='Specify your username (default from Git config)', arg_only=True) +@cli.subcommand('Creates a new keyboard directory') def new_keyboard(cli): - """Creates a new keyboard + """Creates a new keyboard. """ - # TODO: replace this bodge to the existing script - cli.run(['util/new_keyboard.sh'], stdin=None, capture_output=False) + cli.log.info('{style_bright}Generating a new QMK keyboard directory{style_normal}') + cli.echo('') + + # Get keyboard name + new_keyboard_name = None + while not new_keyboard_name: + new_keyboard_name = cli.args.keyboard if cli.args.keyboard else question('Keyboard Name:') + if not validate_keyboard_name(new_keyboard_name): + cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + continue + + keyboard_path = qmk.path.keyboard(new_keyboard_name) + if keyboard_path.exists(): + cli.log.error(f'Keyboard {{fg_cyan}}{new_keyboard_name}{{fg_reset}} already exists! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + + # Get keyboard type + keyboard_type = cli.args.type if cli.args.type else choice('Keyboard Type:', KEYBOARD_TYPES, default=0) + + # Get username + user_name = None + while not user_name: + user_name = question('Your Name:', default=find_user_name()) + + if not user_name: + cli.log.error('You didn\'t provide a username, and we couldn\'t find one set in your QMK or Git configs. Please try again.') + + # Exit if passed by arg + if cli.args.username: + return False + + # Copy all the files + copy_templates(keyboard_type, keyboard_path) + + # Replace all the placeholders + keyboard_basename = keyboard_path.name + replacements = [ + ('%YEAR%', str(date.today().year)), + ('%KEYBOARD%', keyboard_basename), + ('%YOUR_NAME%', user_name), + ] + filenames = [ + keyboard_path / 'config.h', + keyboard_path / 'info.json', + keyboard_path / 'readme.md', + keyboard_path / f'{keyboard_basename}.c', + keyboard_path / f'{keyboard_basename}.h', + keyboard_path / 'keymaps/default/readme.md', + keyboard_path / 'keymaps/default/keymap.c', + ] + replace_placeholders(replacements, filenames) + + cli.echo('') + cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{new_keyboard_name}{{fg_green}}.{{fg_reset}}') + cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}{keyboard_path}{{fg_reset}},') + cli.log.info('or open the directory in your preferred text editor.') + + +def find_user_name(): + if cli.args.username: + return cli.args.username + elif cli.config.user.name: + return cli.config.user.name + else: + return git_get_username() + + +def copy_templates(keyboard_type, keyboard_path): + """Copies the template files from data/templates to the new keyboard directory. + """ + template_base_path = Path('data/templates') + keyboard_basename = keyboard_path.name + + cli.log.info('Copying base template files...') + shutil.copytree(template_base_path / 'base', keyboard_path) + + cli.log.info(f'Copying {{fg_cyan}}{keyboard_type}{{fg_reset}} template files...') + shutil.copytree(template_base_path / keyboard_type, keyboard_path, dirs_exist_ok=True) + + cli.log.info(f'Renaming {{fg_cyan}}keyboard.[ch]{{fg_reset}} to {{fg_cyan}}{keyboard_basename}.[ch]{{fg_reset}}...') + shutil.move(keyboard_path / 'keyboard.c', keyboard_path / f'{keyboard_basename}.c') + shutil.move(keyboard_path / 'keyboard.h', keyboard_path / f'{keyboard_basename}.h') + + +def replace_placeholders(replacements, filenames): + """Replaces the given placeholders in each template file. + """ + for replacement in replacements: + cli.log.info(f'Replacing {{fg_cyan}}{replacement[0]}{{fg_reset}} with {{fg_cyan}}{replacement[1]}{{fg_reset}}...') + + with fileinput.input(files=filenames, inplace=True) as file: + for line in file: + print(line.replace(replacement[0], replacement[1]), end='') diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py index abe5f6de19..c624f74aeb 100755 --- a/lib/python/qmk/cli/pyformat.py +++ b/lib/python/qmk/cli/pyformat.py @@ -1,26 +1,24 @@ -"""Format python code according to QMK's style. +"""Point people to the new command name. """ -from subprocess import CalledProcessError, DEVNULL +import sys +from pathlib import Path from milc import cli -@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") -@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") +@cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True) def pyformat(cli): - """Format python code according to QMK's style. + """Pointer to the new command name: qmk format-python. """ - edit = '--diff' if cli.args.dry_run else '--in-place' - yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] - try: - cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) - cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') - return True + cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('pyformat')] = 'format-python' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') - except CalledProcessError: - if cli.args.dry_run: - cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') - else: - cli.log.error('Error formatting python code!') + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) - return False + return cli.run(argv, capture_output=False).returncode |