diff options
Diffstat (limited to 'lib')
29 files changed, 726 insertions, 276 deletions
diff --git a/lib/chibios b/lib/chibios -Subproject ffe54d63cb10a355add318f8e922e39f1c3d4bf +Subproject 413e39c5681d181720440f2a8b7391f581788d7 diff --git a/lib/chibios-contrib b/lib/chibios-contrib -Subproject 61baa6b036138c155f7cfc5646d833d9423f324 +Subproject ae3542876c928b352dcdaab537dc44ac73d1f45 diff --git a/lib/googletest b/lib/googletest -Subproject ec44c6c1675c25b9827aacd08c02433cccde778 +Subproject e2239ee6043f73722e7aa812a459f54a2855292 diff --git a/lib/lufa b/lib/lufa -Subproject ce10f7642b0459e409839b23cc91498945119b4 +Subproject 19a5d533f02a7b46eeadca99cc9699659cef7a6 diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 1e1c266710..dea0eaeaf9 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -40,7 +40,10 @@ subcommands = [ 'qmk.cli.doctor', 'qmk.cli.fileformat', 'qmk.cli.flash', + 'qmk.cli.format.c', 'qmk.cli.format.json', + 'qmk.cli.format.python', + 'qmk.cli.format.text', 'qmk.cli.generate.api', 'qmk.cli.generate.config_h', 'qmk.cli.generate.dfu_header', @@ -50,6 +53,7 @@ subcommands = [ 'qmk.cli.generate.layouts', 'qmk.cli.generate.rgb_breathe_table', 'qmk.cli.generate.rules_mk', + 'qmk.cli.generate.version_h', 'qmk.cli.hello', 'qmk.cli.info', 'qmk.cli.json2c', diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py index efeb459676..9d0ecaeba3 100644..100755 --- a/lib/python/qmk/cli/cformat.py +++ b/lib/python/qmk/cli/cformat.py @@ -1,137 +1,28 @@ -"""Format C code according to QMK's style. +"""Point people to the new command name. """ -from os import path -from shutil import which -from subprocess import CalledProcessError, DEVNULL, Popen, PIPE +import sys +from pathlib import Path -from argcomplete.completers import FilesCompleter from milc import cli -from qmk.path import normpath -from qmk.c_parse import c_source_files - -c_file_suffixes = ('c', 'h', 'cpp') -core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms') -ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios') - - -def find_clang_format(): - """Returns the path to clang-format. - """ - for clang_version in range(20, 6, -1): - binary = f'clang-format-{clang_version}' - - if which(binary): - return binary - - return 'clang-format' - - -def find_diffs(files): - """Run clang-format and diff it against a file. - """ - found_diffs = False - - for file in files: - cli.log.debug('Checking for changes in %s', file) - clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True) - diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True) - - if diff.returncode != 0: - print(diff.stdout) - found_diffs = True - - return found_diffs - - -def cformat_run(files): - """Spawn clang-format subprocess with proper arguments - """ - # Determine which version of clang-format to use - clang_format = [find_clang_format(), '-i'] - - try: - cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL) - cli.log.info('Successfully formatted the C code.') - return True - - except CalledProcessError as e: - cli.log.error('Error formatting C code!') - cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode) - cli.log.debug('STDOUT:') - cli.log.debug(e.stdout) - cli.log.debug('STDERR:') - cli.log.debug(e.stderr) - return False - - -def filter_files(files, core_only=False): - """Yield only files to be formatted and skip the rest - """ - if core_only: - # Filter non-core files - for index, file in enumerate(files): - # The following statement checks each file to see if the file path is - # - in the core directories - # - not in the ignored directories - if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): - files[index] = None - cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) - - for file in files: - if file and file.name.split('.')[-1] in c_file_suffixes: - yield file - else: - cli.log.debug('Skipping file %s', file) - @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') @cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') @cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') -@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') -@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) +@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.') +@cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True) def cformat(cli): - """Format C code according to QMK's style. + """Pointer to the new command name: qmk format-c. """ - # Find the list of files to format - if cli.args.files: - files = list(filter_files(cli.args.files, cli.args.core_only)) - - if not files: - cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) - exit(0) - - if cli.args.all_files: - cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) - - elif cli.args.all_files: - all_files = c_source_files(core_dirs) - files = list(filter_files(all_files, True)) - - else: - git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs] - git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) - - if git_diff.returncode != 0: - cli.log.error("Error running %s", git_diff_cmd) - print(git_diff.stderr) - return git_diff.returncode - - files = [] - - for file in git_diff.stdout.strip().split('\n'): - if not any([file.startswith(ignore) for ignore in ignored]): - if path.exists(file) and file.split('.')[-1] in c_file_suffixes: - files.append(file) + cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('cformat')] = 'format-c' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') - # Sanity check - if not files: - cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files') - return False + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) - # Run clang-format on the files we've found - if cli.args.dry_run: - return not find_diffs(files) - else: - return cformat_run(files) + return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/doctor/__init__.py b/lib/python/qmk/cli/doctor/__init__.py new file mode 100755 index 0000000000..272e042023 --- /dev/null +++ b/lib/python/qmk/cli/doctor/__init__.py @@ -0,0 +1,5 @@ +"""QMK Doctor + +Check out the user's QMK environment and make sure it's ready to compile. +""" +from .main import doctor diff --git a/lib/python/qmk/os_helpers/__init__.py b/lib/python/qmk/cli/doctor/check.py index 3e98db3c32..0807f41518 100644 --- a/lib/python/qmk/os_helpers/__init__.py +++ b/lib/python/qmk/cli/doctor/check.py @@ -1,4 +1,4 @@ -"""OS-agnostic helper functions +"""Check for specific programs. """ from enum import Enum import re @@ -30,7 +30,7 @@ ESSENTIAL_BINARIES = { } -def parse_gcc_version(version): +def _parse_gcc_version(version): m = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", version) return { @@ -40,7 +40,7 @@ def parse_gcc_version(version): } -def check_arm_gcc_version(): +def _check_arm_gcc_version(): """Returns True if the arm-none-eabi-gcc version is not known to cause problems. """ if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']: @@ -50,7 +50,7 @@ def check_arm_gcc_version(): return CheckStatus.OK # Right now all known arm versions are ok -def check_avr_gcc_version(): +def _check_avr_gcc_version(): """Returns True if the avr-gcc version is not known to cause problems. """ rc = CheckStatus.ERROR @@ -60,7 +60,7 @@ def check_avr_gcc_version(): cli.log.info('Found avr-gcc version %s', version_number) rc = CheckStatus.OK - parsed_version = parse_gcc_version(version_number) + parsed_version = _parse_gcc_version(version_number) if parsed_version['major'] > 8: cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.') rc = CheckStatus.WARNING @@ -68,7 +68,7 @@ def check_avr_gcc_version(): return rc -def check_avrdude_version(): +def _check_avrdude_version(): if 'output' in ESSENTIAL_BINARIES['avrdude']: last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2] version_number = last_line.split()[2][:-1] @@ -77,7 +77,7 @@ def check_avrdude_version(): return CheckStatus.OK -def check_dfu_util_version(): +def _check_dfu_util_version(): if 'output' in ESSENTIAL_BINARIES['dfu-util']: first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0] version_number = first_line.split()[1] @@ -86,7 +86,7 @@ def check_dfu_util_version(): return CheckStatus.OK -def check_dfu_programmer_version(): +def _check_dfu_programmer_version(): if 'output' in ESSENTIAL_BINARIES['dfu-programmer']: first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0] version_number = first_line.split()[1] @@ -111,7 +111,7 @@ def check_binary_versions(): """Check the versions of ESSENTIAL_BINARIES """ versions = [] - for check in (check_arm_gcc_version, check_avr_gcc_version, check_avrdude_version, check_dfu_util_version, check_dfu_programmer_version): + for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version): versions.append(check()) return versions @@ -159,6 +159,6 @@ def check_git_repo(): This is a decent enough indicator that the qmk_firmware directory is a proper Git repository, rather than a .zip download from GitHub. """ - dot_git_dir = QMK_FIRMWARE / '.git' + dot_git = QMK_FIRMWARE / '.git' - return CheckStatus.OK if dot_git_dir.is_dir() else CheckStatus.WARNING + return CheckStatus.OK if dot_git.exists() else CheckStatus.WARNING diff --git a/lib/python/qmk/os_helpers/linux/__init__.py b/lib/python/qmk/cli/doctor/linux.py index 008654ab0f..8ea04cd698 100644 --- a/lib/python/qmk/os_helpers/linux/__init__.py +++ b/lib/python/qmk/cli/doctor/linux.py @@ -1,11 +1,13 @@ """OS-specific functions for: Linux """ -from pathlib import Path +import platform import shutil +from pathlib import Path from milc import cli + from qmk.constants import QMK_FIRMWARE -from qmk.os_helpers import CheckStatus +from .check import CheckStatus def _udev_rule(vid, pid=None, *args): @@ -39,7 +41,12 @@ def check_udev_rules(): """Make sure the udev rules look good. """ rc = CheckStatus.OK - udev_dir = Path("/etc/udev/rules.d/") + udev_dirs = [ + Path("/usr/lib/udev/rules.d/"), + Path("/usr/local/lib/udev/rules.d/"), + Path("/run/udev/rules.d/"), + Path("/etc/udev/rules.d/"), + ] desired_rules = { 'atmel-dfu': { _udev_rule("03eb", "2fef"), # ATmega16U2 @@ -88,8 +95,8 @@ def check_udev_rules(): 'tmk': {_deprecated_udev_rule("feed")} } - if udev_dir.exists(): - udev_rules = [rule_file for rule_file in udev_dir.glob('*.rules')] + if any(udev_dir.exists() for udev_dir in udev_dirs): + udev_rules = [rule_file for udev_dir in udev_dirs for rule_file in udev_dir.glob('*.rules')] current_rules = set() # Collect all rules from the config files @@ -115,7 +122,8 @@ def check_udev_rules(): cli.log.warning("{fg_yellow}Missing or outdated udev rules for '%s' boards. Run 'sudo cp %s/util/udev/50-qmk.rules /etc/udev/rules.d/'.", bootloader, QMK_FIRMWARE) else: - cli.log.warning("{fg_yellow}'%s' does not exist. Skipping udev rule checking...", udev_dir) + cli.log.warning("{fg_yellow}Can't find udev rules, skipping udev rule checking...") + cli.log.debug("Checked directories: %s", ', '.join(str(udev_dir) for udev_dir in udev_dirs)) return rc @@ -138,3 +146,23 @@ def check_modem_manager(): """(TODO): Add check for non-systemd systems """ return False + + +def os_test_linux(): + """Run the Linux specific tests. + """ + # Don't bother with udev on WSL, for now + if 'microsoft' in platform.uname().release.lower(): + cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.") + + # https://github.com/microsoft/WSL/issues/4197 + if QMK_FIRMWARE.as_posix().startswith("/mnt"): + cli.log.warning("I/O performance on /mnt may be extremely slow.") + return CheckStatus.WARNING + + return CheckStatus.OK + else: + cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") + from .linux import check_udev_rules + + return check_udev_rules() diff --git a/lib/python/qmk/cli/doctor/macos.py b/lib/python/qmk/cli/doctor/macos.py new file mode 100644 index 0000000000..00fb272858 --- /dev/null +++ b/lib/python/qmk/cli/doctor/macos.py @@ -0,0 +1,13 @@ +import platform + +from milc import cli + +from .check import CheckStatus + + +def os_test_macos(): + """Run the Mac specific tests. + """ + cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) + + return CheckStatus.OK diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor/main.py index 327bc9cb30..6a31ccdfdd 100755 --- a/lib/python/qmk/cli/doctor.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -7,9 +7,11 @@ from subprocess import DEVNULL from milc import cli from milc.questions import yesno + from qmk import submodules -from qmk.constants import QMK_FIRMWARE -from qmk.os_helpers import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules +from qmk.commands import git_check_repo, git_get_branch, git_is_dirty, git_get_remotes, git_check_deviation, in_virtualenv def os_tests(): @@ -18,51 +20,48 @@ def os_tests(): platform_id = platform.platform().lower() if 'darwin' in platform_id or 'macos' in platform_id: + from .macos import os_test_macos return os_test_macos() elif 'linux' in platform_id: + from .linux import os_test_linux return os_test_linux() elif 'windows' in platform_id: + from .windows import os_test_windows return os_test_windows() else: cli.log.warning('Unsupported OS detected: %s', platform_id) return CheckStatus.WARNING -def os_test_linux(): - """Run the Linux specific tests. +def git_tests(): + """Run Git-related checks """ - # Don't bother with udev on WSL, for now - if 'microsoft' in platform.uname().release.lower(): - cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.") - - # https://github.com/microsoft/WSL/issues/4197 - if QMK_FIRMWARE.as_posix().startswith("/mnt"): - cli.log.warning("I/O performance on /mnt may be extremely slow.") - return CheckStatus.WARNING + status = CheckStatus.OK - return CheckStatus.OK + # Make sure our QMK home is a Git repo + git_ok = git_check_repo() + if not git_ok: + cli.log.warning("{fg_yellow}QMK home does not appear to be a Git repository! (no .git folder)") + status = CheckStatus.WARNING else: - cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") - from qmk.os_helpers.linux import check_udev_rules - - return check_udev_rules() - - -def os_test_macos(): - """Run the Mac specific tests. - """ - cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) - - return CheckStatus.OK - - -def os_test_windows(): - """Run the Windows specific tests. - """ - win32_ver = platform.win32_ver() - cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1]) - - return CheckStatus.OK + git_branch = git_get_branch() + if git_branch: + cli.log.info('Git branch: %s', git_branch) + git_dirty = git_is_dirty() + if git_dirty: + cli.log.warning('{fg_yellow}Git has unstashed/uncommitted changes.') + status = CheckStatus.WARNING + git_remotes = git_get_remotes() + if 'upstream' not in git_remotes.keys() or QMK_FIRMWARE_UPSTREAM not in git_remotes['upstream'].get('url', ''): + cli.log.warning('{fg_yellow}The official repository does not seem to be configured as git remote "upstream".') + status = CheckStatus.WARNING + else: + git_deviation = git_check_deviation(git_branch) + if git_branch in ['master', 'develop'] and git_deviation: + cli.log.warning('{fg_yellow}The local "%s" branch contains commits not found in the upstream branch.', git_branch) + status = CheckStatus.WARNING + + return status @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') @@ -82,12 +81,11 @@ def doctor(cli): status = os_tests() - # Make sure our QMK home is a Git repo - git_ok = check_git_repo() + status = git_tests() - if git_ok == CheckStatus.WARNING: - cli.log.warning("QMK home does not appear to be a Git repository! (no .git folder)") - status = CheckStatus.WARNING + venv = in_virtualenv() + if venv: + cli.log.info('CLI installed in virtualenv.') # Make sure the basic CLI tools we need are available and can be executed. bin_ok = check_binaries() diff --git a/lib/python/qmk/cli/doctor/windows.py b/lib/python/qmk/cli/doctor/windows.py new file mode 100644 index 0000000000..381ab36fde --- /dev/null +++ b/lib/python/qmk/cli/doctor/windows.py @@ -0,0 +1,14 @@ +import platform + +from milc import cli + +from .check import CheckStatus + + +def os_test_windows(): + """Run the Windows specific tests. + """ + win32_ver = platform.win32_ver() + cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1]) + + return CheckStatus.OK diff --git a/lib/python/qmk/cli/fileformat.py b/lib/python/qmk/cli/fileformat.py index 112d8d59da..cee4ba1acd 100644..100755 --- a/lib/python/qmk/cli/fileformat.py +++ b/lib/python/qmk/cli/fileformat.py @@ -1,13 +1,23 @@ -"""Format files according to QMK's style. +"""Point people to the new command name. """ -from milc import cli +import sys +from pathlib import Path -import subprocess +from milc import cli -@cli.subcommand("Format files according to QMK's style.", hidden=True) +@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True) def fileformat(cli): - """Run several general formatting commands. + """Pointer to the new command name: qmk format-text. """ - dos2unix = subprocess.run(['bash', '-c', 'git ls-files -z | xargs -0 dos2unix'], stdout=subprocess.DEVNULL) - return dos2unix.returncode + cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('fileformat')] = 'format-text' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') + + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) + + return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/format/c.py b/lib/python/qmk/cli/format/c.py new file mode 100644 index 0000000000..b7263e19f3 --- /dev/null +++ b/lib/python/qmk/cli/format/c.py @@ -0,0 +1,137 @@ +"""Format C code according to QMK's style. +""" +from os import path +from shutil import which +from subprocess import CalledProcessError, DEVNULL, Popen, PIPE + +from argcomplete.completers import FilesCompleter +from milc import cli + +from qmk.path import normpath +from qmk.c_parse import c_source_files + +c_file_suffixes = ('c', 'h', 'cpp') +core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms') +ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios') + + +def find_clang_format(): + """Returns the path to clang-format. + """ + for clang_version in range(20, 6, -1): + binary = f'clang-format-{clang_version}' + + if which(binary): + return binary + + return 'clang-format' + + +def find_diffs(files): + """Run clang-format and diff it against a file. + """ + found_diffs = False + + for file in files: + cli.log.debug('Checking for changes in %s', file) + clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True) + diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True) + + if diff.returncode != 0: + print(diff.stdout) + found_diffs = True + + return found_diffs + + +def cformat_run(files): + """Spawn clang-format subprocess with proper arguments + """ + # Determine which version of clang-format to use + clang_format = [find_clang_format(), '-i'] + + try: + cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL) + cli.log.info('Successfully formatted the C code.') + return True + + except CalledProcessError as e: + cli.log.error('Error formatting C code!') + cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode) + cli.log.debug('STDOUT:') + cli.log.debug(e.stdout) + cli.log.debug('STDERR:') + cli.log.debug(e.stderr) + return False + + +def filter_files(files, core_only=False): + """Yield only files to be formatted and skip the rest + """ + if core_only: + # Filter non-core files + for index, file in enumerate(files): + # The following statement checks each file to see if the file path is + # - in the core directories + # - not in the ignored directories + if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): + files[index] = None + cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) + + for file in files: + if file and file.name.split('.')[-1] in c_file_suffixes: + yield file + else: + cli.log.debug('Skipping file %s', file) + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") +@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') +@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') +@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') +@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') +@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) +def format_c(cli): + """Format C code according to QMK's style. + """ + # Find the list of files to format + if cli.args.files: + files = list(filter_files(cli.args.files, cli.args.core_only)) + + if not files: + cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) + exit(0) + + if cli.args.all_files: + cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) + + elif cli.args.all_files: + all_files = c_source_files(core_dirs) + files = list(filter_files(all_files, True)) + + else: + git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs] + git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) + + if git_diff.returncode != 0: + cli.log.error("Error running %s", git_diff_cmd) + print(git_diff.stderr) + return git_diff.returncode + + files = [] + + for file in git_diff.stdout.strip().split('\n'): + if not any([file.startswith(ignore) for ignore in ignored]): + if path.exists(file) and file.split('.')[-1] in c_file_suffixes: + files.append(file) + + # Sanity check + if not files: + cli.log.error('No changed files detected. Use "qmk format-c -a" to format all core files') + return False + + # Run clang-format on the files we've found + if cli.args.dry_run: + return not find_diffs(files) + else: + return cformat_run(files) diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 1358c70e7a..19d504491f 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -8,7 +8,7 @@ from jsonschema import ValidationError from milc import cli from qmk.info import info_json -from qmk.json_schema import json_load, keyboard_validate +from qmk.json_schema import json_load, validate from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder from qmk.path import normpath @@ -23,14 +23,13 @@ def format_json(cli): if cli.args.format == 'auto': try: - keyboard_validate(json_file) + validate(json_file, 'qmk.keyboard.v1') json_encoder = InfoJSONEncoder except ValidationError as e: cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) cli.log.info('Treating %s as a keymap file.', cli.args.json_file) json_encoder = KeymapJSONEncoder - elif cli.args.format == 'keyboard': json_encoder = InfoJSONEncoder elif cli.args.format == 'keymap': diff --git a/lib/python/qmk/cli/format/python.py b/lib/python/qmk/cli/format/python.py new file mode 100755 index 0000000000..00612f97ec --- /dev/null +++ b/lib/python/qmk/cli/format/python.py @@ -0,0 +1,26 @@ +"""Format python code according to QMK's style. +""" +from subprocess import CalledProcessError, DEVNULL + +from milc import cli + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") +@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) +def format_python(cli): + """Format python code according to QMK's style. + """ + edit = '--diff' if cli.args.dry_run else '--in-place' + yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] + try: + cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) + cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') + return True + + except CalledProcessError: + if cli.args.dry_run: + cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') + else: + cli.log.error('Error formatting python code!') + + return False diff --git a/lib/python/qmk/cli/format/text.py b/lib/python/qmk/cli/format/text.py new file mode 100644 index 0000000000..e7e07b7297 --- /dev/null +++ b/lib/python/qmk/cli/format/text.py @@ -0,0 +1,27 @@ +"""Ensure text files have the proper line endings. +""" +from subprocess import CalledProcessError + +from milc import cli + + +@cli.subcommand("Ensure text files have the proper line endings.", hidden=True) +def format_text(cli): + """Ensure text files have the proper line endings. + """ + try: + file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True) + except CalledProcessError as e: + cli.log.error('Could not get file list: %s', e) + exit(1) + except Exception as e: + cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e) + cli.log.exception(e) + exit(1) + + dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout) + + if dos2unix.returncode != 0: + print(dos2unix.stderr) + + return dos2unix.returncode diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py new file mode 100644 index 0000000000..b8e52588c4 --- /dev/null +++ b/lib/python/qmk/cli/generate/version_h.py @@ -0,0 +1,28 @@ +"""Used by the make system to generate version.h for use in code. +""" +from milc import cli + +from qmk.commands import create_version_h +from qmk.path import normpath + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('--skip-git', arg_only=True, action='store_true', help='Skip Git operations') +@cli.argument('--skip-all', arg_only=True, action='store_true', help='Use placeholder values for all defines (implies --skip-git)') +@cli.subcommand('Used by the make system to generate version.h for use in code', hidden=True) +def generate_version_h(cli): + """Generates the version.h file. + """ + if cli.args.skip_all: + cli.args.skip_git = True + + version_h = create_version_h(cli.args.skip_git, cli.args.skip_all) + + if cli.args.output: + cli.args.output.write_text(version_h) + + if not cli.args.quiet: + cli.log.info('Wrote version.h to %s.', cli.args.output) + else: + print(version_h) diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py index 0d08d242cd..337b494a99 100755 --- a/lib/python/qmk/cli/info.py +++ b/lib/python/qmk/cli/info.py @@ -87,8 +87,6 @@ def print_friendly_output(kb_info_json): cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer']) cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown')) cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys()))) - if 'width' in kb_info_json and 'height' in kb_info_json: - cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height'])) cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown')) cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown')) if 'layout_aliases' in kb_info_json: diff --git a/lib/python/qmk/cli/kle2json.py b/lib/python/qmk/cli/kle2json.py index acb75ef4fd..bbfddf4268 100755 --- a/lib/python/qmk/cli/kle2json.py +++ b/lib/python/qmk/cli/kle2json.py @@ -44,8 +44,6 @@ def kle2json(cli): 'keyboard_name': kle.name, 'url': '', 'maintainer': 'qmk', - 'width': kle.columns, - 'height': kle.rows, 'layouts': { 'LAYOUT': { 'layout': kle2qmk(kle) diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index ae4445ca48..9e4232679d 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -1,11 +1,142 @@ -"""This script automates the creation of keyboards. +"""This script automates the creation of new keyboard directories using a starter template. """ +from datetime import date +import fileinput +from pathlib import Path +import re +import shutil + +from qmk.commands import git_get_username +import qmk.path from milc import cli +from milc.questions import choice, question + +KEYBOARD_TYPES = ['avr', 'ps2avrgb'] + + +def keyboard_name(name): + """Callable for argparse validation. + """ + if not validate_keyboard_name(name): + raise ValueError + return name -@cli.subcommand('Creates a new keyboard') +def validate_keyboard_name(name): + """Returns True if the given keyboard name contains only lowercase a-z, 0-9 and underscore characters. + """ + regex = re.compile(r'^[a-z0-9][a-z0-9/_]+$') + return bool(regex.match(name)) + + +@cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name) +@cli.argument('-t', '--type', help='Specify the keyboard type', arg_only=True, choices=KEYBOARD_TYPES) +@cli.argument('-u', '--username', help='Specify your username (default from Git config)', arg_only=True) +@cli.subcommand('Creates a new keyboard directory') def new_keyboard(cli): - """Creates a new keyboard + """Creates a new keyboard. """ - # TODO: replace this bodge to the existing script - cli.run(['util/new_keyboard.sh'], stdin=None, capture_output=False) + cli.log.info('{style_bright}Generating a new QMK keyboard directory{style_normal}') + cli.echo('') + + # Get keyboard name + new_keyboard_name = None + while not new_keyboard_name: + new_keyboard_name = cli.args.keyboard if cli.args.keyboard else question('Keyboard Name:') + if not validate_keyboard_name(new_keyboard_name): + cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + continue + + keyboard_path = qmk.path.keyboard(new_keyboard_name) + if keyboard_path.exists(): + cli.log.error(f'Keyboard {{fg_cyan}}{new_keyboard_name}{{fg_reset}} already exists! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + + # Get keyboard type + keyboard_type = cli.args.type if cli.args.type else choice('Keyboard Type:', KEYBOARD_TYPES, default=0) + + # Get username + user_name = None + while not user_name: + user_name = question('Your Name:', default=find_user_name()) + + if not user_name: + cli.log.error('You didn\'t provide a username, and we couldn\'t find one set in your QMK or Git configs. Please try again.') + + # Exit if passed by arg + if cli.args.username: + return False + + # Copy all the files + copy_templates(keyboard_type, keyboard_path) + + # Replace all the placeholders + keyboard_basename = keyboard_path.name + replacements = [ + ('%YEAR%', str(date.today().year)), + ('%KEYBOARD%', keyboard_basename), + ('%YOUR_NAME%', user_name), + ] + filenames = [ + keyboard_path / 'config.h', + keyboard_path / 'info.json', + keyboard_path / 'readme.md', + keyboard_path / f'{keyboard_basename}.c', + keyboard_path / f'{keyboard_basename}.h', + keyboard_path / 'keymaps/default/readme.md', + keyboard_path / 'keymaps/default/keymap.c', + ] + replace_placeholders(replacements, filenames) + + cli.echo('') + cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{new_keyboard_name}{{fg_green}}.{{fg_reset}}') + cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}{keyboard_path}{{fg_reset}},') + cli.log.info('or open the directory in your preferred text editor.') + + +def find_user_name(): + if cli.args.username: + return cli.args.username + elif cli.config.user.name: + return cli.config.user.name + else: + return git_get_username() + + +def copy_templates(keyboard_type, keyboard_path): + """Copies the template files from quantum/template to the new keyboard directory. + """ + template_base_path = Path('quantum/template') + keyboard_basename = keyboard_path.name + + cli.log.info('Copying base template files...') + shutil.copytree(template_base_path / 'base', keyboard_path) + + cli.log.info(f'Copying {{fg_cyan}}{keyboard_type}{{fg_reset}} template files...') + shutil.copytree(template_base_path / keyboard_type, keyboard_path, dirs_exist_ok=True) + + cli.log.info(f'Renaming {{fg_cyan}}keyboard.[ch]{{fg_reset}} to {{fg_cyan}}{keyboard_basename}.[ch]{{fg_reset}}...') + shutil.move(keyboard_path / 'keyboard.c', keyboard_path / f'{keyboard_basename}.c') + shutil.move(keyboard_path / 'keyboard.h', keyboard_path / f'{keyboard_basename}.h') + + +def replace_placeholders(replacements, filenames): + """Replaces the given placeholders in each template file. + """ + for replacement in replacements: + cli.log.info(f'Replacing {{fg_cyan}}{replacement[0]}{{fg_reset}} with {{fg_cyan}}{replacement[1]}{{fg_reset}}...') + + with fileinput.input(files=filenames, inplace=True) as file: + for line in file: + print(line.replace(replacement[0], replacement[1]), end='') diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py index abe5f6de19..c624f74aeb 100755 --- a/lib/python/qmk/cli/pyformat.py +++ b/lib/python/qmk/cli/pyformat.py @@ -1,26 +1,24 @@ -"""Format python code according to QMK's style. +"""Point people to the new command name. """ -from subprocess import CalledProcessError, DEVNULL +import sys +from pathlib import Path from milc import cli -@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") -@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") +@cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True) def pyformat(cli): - """Format python code according to QMK's style. + """Pointer to the new command name: qmk format-python. """ - edit = '--diff' if cli.args.dry_run else '--in-place' - yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] - try: - cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) - cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') - return True + cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.') + argv = [sys.executable, *sys.argv] + argv[argv.index('pyformat')] = 'format-python' + script_path = Path(argv[1]) + script_path_exe = Path(f'{argv[1]}.exe') - except CalledProcessError: - if cli.args.dry_run: - cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') - else: - cli.log.error('Error formatting python code!') + if not script_path.exists() and script_path_exe.exists(): + # For reasons I don't understand ".exe" is stripped from the script name on windows. + argv[1] = str(script_path_exe) - return False + return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3a35c11031..8c66228b2b 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -2,6 +2,7 @@ """ import json import os +import sys import shutil from pathlib import Path from subprocess import DEVNULL @@ -10,7 +11,7 @@ from time import strftime from milc import cli import qmk.keymap -from qmk.constants import KEYBOARD_OUTPUT_PREFIX +from qmk.constants import QMK_FIRMWARE, KEYBOARD_OUTPUT_PREFIX from qmk.json_schema import json_load time_fmt = '%Y-%m-%d-%H:%M:%S' @@ -86,11 +87,17 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): return create_make_target(':'.join(make_args), parallel, **env_vars) -def get_git_version(repo_dir='.', check_dir='.'): +def get_git_version(current_time, repo_dir='.', check_dir='.'): """Returns the current git version for a repo, or the current time. """ git_describe_cmd = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags'] + if repo_dir != '.': + repo_dir = Path('lib') / repo_dir + + if check_dir != '.': + check_dir = repo_dir / check_dir + if Path(check_dir).exists(): git_describe = cli.run(git_describe_cmd, stdin=DEVNULL, cwd=repo_dir) @@ -100,23 +107,40 @@ def get_git_version(repo_dir='.', check_dir='.'): else: cli.log.warn(f'"{" ".join(git_describe_cmd)}" returned error code {git_describe.returncode}') print(git_describe.stderr) - return strftime(time_fmt) + return current_time - return strftime(time_fmt) + return current_time -def write_version_h(git_version, build_date, chibios_version, chibios_contrib_version): - """Generate and write quantum/version.h +def create_version_h(skip_git=False, skip_all=False): + """Generate version.h contents """ - version_h = [ - f'#define QMK_VERSION "{git_version}"', - f'#define QMK_BUILDDATE "{build_date}"', - f'#define CHIBIOS_VERSION "{chibios_version}"', - f'#define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}"', - ] + if skip_all: + current_time = "1970-01-01-00:00:00" + else: + current_time = strftime(time_fmt) + + if skip_git: + git_version = "NA" + chibios_version = "NA" + chibios_contrib_version = "NA" + else: + git_version = get_git_version(current_time) + chibios_version = get_git_version(current_time, "chibios", "os") + chibios_contrib_version = get_git_version(current_time, "chibios-contrib", "os") + + version_h_lines = f"""/* This file was automatically generated. Do not edit or copy. + */ + +#pragma once + +#define QMK_VERSION "{git_version}" +#define QMK_BUILDDATE "{current_time}" +#define CHIBIOS_VERSION "{chibios_version}" +#define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}" +""" - version_h_file = Path('quantum/version.h') - version_h_file.write_text('\n'.join(version_h)) + return version_h_lines def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_vars): @@ -149,13 +173,8 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va keymap_dir.mkdir(exist_ok=True, parents=True) keymap_c.write_text(c_text) - # Write the version.h file - git_version = get_git_version() - build_date = strftime('%Y-%m-%d-%H:%M:%S') - chibios_version = get_git_version("lib/chibios", "lib/chibios/os") - chibios_contrib_version = get_git_version("lib/chibios-contrib", "lib/chibios-contrib/os") - - write_version_h(git_version, build_date, chibios_version, chibios_contrib_version) + version_h = Path('quantum/version.h') + version_h.write_text(create_version_h()) # Return a command that can be run to make the keymap and flash if given verbose = 'true' if cli.config.general.verbose else 'false' @@ -181,10 +200,6 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va make_command.append(f'{key}={value}') make_command.extend([ - f'GIT_VERSION={git_version}', - f'BUILD_DATE={build_date}', - f'CHIBIOS_VERSION={chibios_version}', - f'CHIBIOS_CONTRIB_VERSION={chibios_contrib_version}', f'KEYBOARD={user_keymap["keyboard"]}', f'KEYMAP={user_keymap["keymap"]}', f'KEYBOARD_FILESAFE={keyboard_filesafe}', @@ -223,3 +238,80 @@ def parse_configurator_json(configurator_file): user_keymap['layout'] = aliases[orig_keyboard]['layouts'][user_keymap['layout']] return user_keymap + + +def git_get_username(): + """Retrieves user's username from Git config, if set. + """ + git_username = cli.run(['git', 'config', '--get', 'user.name']) + + if git_username.returncode == 0 and git_username.stdout: + return git_username.stdout.strip() + + +def git_check_repo(): + """Checks that the .git directory exists inside QMK_HOME. + + This is a decent enough indicator that the qmk_firmware directory is a + proper Git repository, rather than a .zip download from GitHub. + """ + dot_git_dir = QMK_FIRMWARE / '.git' + + return dot_git_dir.is_dir() + + +def git_get_branch(): + """Returns the current branch for a repo, or None. + """ + git_branch = cli.run(['git', 'branch', '--show-current']) + if not git_branch.returncode != 0 or not git_branch.stdout: + # Workaround for Git pre-2.22 + git_branch = cli.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + + if git_branch.returncode == 0: + return git_branch.stdout.strip() + + +def git_is_dirty(): + """Returns 1 if repo is dirty, or 0 if clean + """ + git_diff_staged_cmd = ['git', 'diff', '--quiet'] + git_diff_unstaged_cmd = [*git_diff_staged_cmd, '--cached'] + + unstaged = cli.run(git_diff_staged_cmd) + staged = cli.run(git_diff_unstaged_cmd) + + return unstaged.returncode != 0 or staged.returncode != 0 + + +def git_get_remotes(): + """Returns the current remotes for a repo. + """ + remotes = {} + + git_remote_show_cmd = ['git', 'remote', 'show'] + git_remote_get_cmd = ['git', 'remote', 'get-url'] + + git_remote_show = cli.run(git_remote_show_cmd) + if git_remote_show.returncode == 0: + for name in git_remote_show.stdout.splitlines(): + git_remote_name = cli.run([*git_remote_get_cmd, name]) + remotes[name.strip()] = {"url": git_remote_name.stdout.strip()} + + return remotes + + +def git_check_deviation(active_branch): + """Return True if branch has custom commits + """ + cli.run(['git', 'fetch', 'upstream', active_branch]) + deviations = cli.run(['git', '--no-pager', 'log', f'upstream/{active_branch}...{active_branch}']) + return bool(deviations.returncode) + + +def in_virtualenv(): + """Check if running inside a virtualenv. + Based on https://stackoverflow.com/a/1883251 + """ + active_prefix = getattr(sys, "base_prefix", None) or getattr(sys, "real_prefix", None) or sys.prefix + return active_prefix != sys.prefix diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 49e5e0eb42..1078f4ad5e 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -6,11 +6,14 @@ from pathlib import Path # The root of the qmk_firmware tree. QMK_FIRMWARE = Path.cwd() +# Upstream repo url +QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' + # This is the number of directories under `qmk_firmware/keyboards` that will be traversed. This is currently a limitation of our make system. MAX_KEYBOARD_SUBFOLDERS = 5 # Supported processor types -CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66F18', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L433', 'STM32L443' +CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66F18', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L433', 'STM32L443' LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85' diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 47c8bff7a8..bcb4d81ef2 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -9,7 +9,7 @@ from milc import cli from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS from qmk.c_parse import find_layouts -from qmk.json_schema import deep_update, json_load, keyboard_validate, keyboard_api_validate +from qmk.json_schema import deep_update, json_load, validate from qmk.keyboard import config_h, rules_mk from qmk.keymap import list_keymaps from qmk.makefile import parse_rules_mk_file @@ -64,9 +64,12 @@ def info_json(keyboard): info_data = _extract_config_h(info_data) info_data = _extract_rules_mk(info_data) + # Ensure that we have matrix row and column counts + info_data = _matrix_size(info_data) + # Validate against the jsonschema try: - keyboard_api_validate(info_data) + validate(info_data, 'qmk.api.keyboard.v1') except jsonschema.ValidationError as e: json_path = '.'.join([str(p) for p in e.absolute_path]) @@ -90,6 +93,9 @@ def info_json(keyboard): if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}): _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name)) + # Check that the reported matrix size is consistent with the actual matrix size + _check_matrix(info_data) + return info_data @@ -143,10 +149,7 @@ def _pin_name(pin): elif pin == 'NO_PIN': return None - elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit(): - return pin - - raise ValueError(f'Invalid pin: {pin}') + return pin def _extract_pins(pins): @@ -341,6 +344,46 @@ def _extract_rules_mk(info_data): return info_data +def _matrix_size(info_data): + """Add info_data['matrix_size'] if it doesn't exist. + """ + if 'matrix_size' not in info_data and 'matrix_pins' in info_data: + info_data['matrix_size'] = {} + + if 'direct' in info_data['matrix_pins']: + info_data['matrix_size']['cols'] = len(info_data['matrix_pins']['direct'][0]) + info_data['matrix_size']['rows'] = len(info_data['matrix_pins']['direct']) + elif 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']: + info_data['matrix_size']['cols'] = len(info_data['matrix_pins']['cols']) + info_data['matrix_size']['rows'] = len(info_data['matrix_pins']['rows']) + + return info_data + + +def _check_matrix(info_data): + """Check the matrix to ensure that row/column count is consistent. + """ + if 'matrix_pins' in info_data and 'matrix_size' in info_data: + actual_col_count = info_data['matrix_size'].get('cols', 0) + actual_row_count = info_data['matrix_size'].get('rows', 0) + col_count = row_count = 0 + + if 'direct' in info_data['matrix_pins']: + col_count = len(info_data['matrix_pins']['direct'][0]) + row_count = len(info_data['matrix_pins']['direct']) + elif 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']: + col_count = len(info_data['matrix_pins']['cols']) + row_count = len(info_data['matrix_pins']['rows']) + + if col_count != actual_col_count and col_count != (actual_col_count / 2): + # FIXME: once we can we should detect if split is enabled to do the actual_col_count/2 check. + _log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}') + + if row_count != actual_row_count and row_count != (actual_row_count / 2): + # FIXME: once we can we should detect if split is enabled to do the actual_row_count/2 check. + _log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}') + + def _merge_layouts(info_data, new_info_data): """Merge new_info_data into info_data in an intelligent way. """ @@ -493,7 +536,7 @@ def merge_info_jsons(keyboard, info_data): continue try: - keyboard_validate(new_info_data) + validate(new_info_data, 'qmk.keyboard.v1') except jsonschema.ValidationError as e: json_path = '.'.join([str(p) for p in e.absolute_path]) cli.log.error('Not including data from file: %s', info_file) diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py index 9f3da022b4..72e91973a3 100755 --- a/lib/python/qmk/json_encoders.py +++ b/lib/python/qmk/json_encoders.py @@ -102,9 +102,6 @@ class InfoJSONEncoder(QMKJSONEncoder): elif key == 'maintainer': return '12maintainer' - elif key in ('height', 'width'): - return '40' + str(key) - elif key == 'community_layouts': return '97community_layouts' diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py index 077dfcaa93..3e5663a291 100644 --- a/lib/python/qmk/json_schema.py +++ b/lib/python/qmk/json_schema.py @@ -24,9 +24,10 @@ def json_load(json_file): def load_jsonschema(schema_name): """Read a jsonschema file from disk. - - FIXME(skullydazed/anyone): Refactor to make this a public function. """ + if Path(schema_name).exists(): + return json_load(schema_name) + schema_path = Path(f'data/schemas/{schema_name}.jsonschema') if not schema_path.exists(): @@ -35,28 +36,33 @@ def load_jsonschema(schema_name): return json_load(schema_path) -def keyboard_validate(data): - """Validates data against the keyboard jsonschema. +def create_validator(schema): + """Creates a validator for the given schema id. """ - schema = load_jsonschema('keyboard') - validator = jsonschema.Draft7Validator(schema).validate + schema_store = {} - return validator(data) + for schema_file in Path('data/schemas').glob('*.jsonschema'): + schema_data = load_jsonschema(schema_file) + if not isinstance(schema_data, dict): + cli.log.debug('Skipping schema file %s', schema_file) + continue + schema_store[schema_data['$id']] = schema_data + + resolver = jsonschema.RefResolver.from_schema(schema_store['qmk.keyboard.v1'], store=schema_store) + + return jsonschema.Draft7Validator(schema_store[schema], resolver=resolver).validate -def keyboard_api_validate(data): - """Validates data against the api_keyboard jsonschema. +def validate(data, schema): + """Validates data against a schema. """ - base = load_jsonschema('keyboard') - relative = load_jsonschema('api_keyboard') - resolver = jsonschema.RefResolver.from_schema(base) - validator = jsonschema.Draft7Validator(relative, resolver=resolver).validate + validator = create_validator(schema) return validator(data) def deep_update(origdict, newdict): - """Update a dictionary in place, recursing to do a deep copy. + """Update a dictionary in place, recursing to do a depth-first deep copy. """ for key, value in newdict.items(): if isinstance(value, Mapping): diff --git a/lib/python/qmk/tests/minimal_info.json b/lib/python/qmk/tests/minimal_info.json index b91c23bd3d..11ef12fefe 100644 --- a/lib/python/qmk/tests/minimal_info.json +++ b/lib/python/qmk/tests/minimal_info.json @@ -1,8 +1,6 @@ { "keyboard_name": "tester", "maintainer": "qmk", - "height": 5, - "width": 15, "layouts": { "LAYOUT": { "layout": [ diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index afdbc81429..b39fe5e46d 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -31,13 +31,13 @@ def check_returncode(result, expected=[0]): assert result.returncode in expected -def test_cformat(): - result = check_subcommand('cformat', '-n', 'quantum/matrix.c') +def test_format_c(): + result = check_subcommand('format-c', '-n', 'quantum/matrix.c') check_returncode(result) -def test_cformat_all(): - result = check_subcommand('cformat', '-n', '-a') +def test_format_c_all(): + result = check_subcommand('format-c', '-n', '-a') check_returncode(result, [0, 1]) @@ -80,8 +80,8 @@ def test_hello(): assert 'Hello,' in result.stdout -def test_pyformat(): - result = check_subcommand('pyformat', '--dry-run') +def test_format_python(): + result = check_subcommand('format-python', '--dry-run') check_returncode(result) assert 'Python code in `bin/qmk` and `lib/python` is correctly formatted.' in result.stdout @@ -258,6 +258,12 @@ def test_generate_rules_mk(): assert 'MCU ?= atmega32u4' in result.stdout +def test_generate_version_h(): + result = check_subcommand('generate-version-h') + check_returncode(result) + assert '#define QMK_VERSION' in result.stdout + + def test_generate_layouts(): result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic') check_returncode(result) @@ -267,7 +273,7 @@ def test_generate_layouts(): def test_format_json_keyboard(): result = check_subcommand('format-json', '--format', 'keyboard', 'lib/python/qmk/tests/minimal_info.json') check_returncode(result) - assert result.stdout == '{\n "keyboard_name": "tester",\n "maintainer": "qmk",\n "height": 5,\n "width": 15,\n "layouts": {\n "LAYOUT": {\n "layout": [\n { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n ]\n }\n }\n}\n' + assert result.stdout == '{\n "keyboard_name": "tester",\n "maintainer": "qmk",\n "layouts": {\n "LAYOUT": {\n "layout": [\n { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n ]\n }\n }\n}\n' def test_format_json_keymap(): @@ -279,7 +285,7 @@ def test_format_json_keymap(): def test_format_json_keyboard_auto(): result = check_subcommand('format-json', '--format', 'auto', 'lib/python/qmk/tests/minimal_info.json') check_returncode(result) - assert result.stdout == '{\n "keyboard_name": "tester",\n "maintainer": "qmk",\n "height": 5,\n "width": 15,\n "layouts": {\n "LAYOUT": {\n "layout": [\n { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n ]\n }\n }\n}\n' + assert result.stdout == '{\n "keyboard_name": "tester",\n "maintainer": "qmk",\n "layouts": {\n "LAYOUT": {\n "layout": [\n { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 }\n ]\n }\n }\n}\n' def test_format_json_keymap_auto(): |