diff options
author | Drashna Jael're <drashna@live.com> | 2021-12-07 09:27:44 -0800 |
---|---|---|
committer | Drashna Jael're <drashna@live.com> | 2021-12-07 09:27:44 -0800 |
commit | 7c18b1c9d3d968ded45e072af3483547c3ec7859 (patch) | |
tree | ed7c8a4176033046eacff21228364290b44fcfdb /lib/python | |
parent | 43002bdf77ab0f48af6b04e87edcc37f7cb7b905 (diff) | |
parent | 6d0a62920410f50d7f6707960ca1ca0c8fd1d1fa (diff) |
Merge commit '6d0a62920410f50d7f6707960ca1ca0c8fd1d1fa' into firmware21
Diffstat (limited to 'lib/python')
26 files changed, 727 insertions, 476 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 539d03e2fc..c51eece955 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -31,11 +31,11 @@ safe_commands = [ subcommands = [ 'qmk.cli.bux', 'qmk.cli.c2json', + 'qmk.cli.cd', 'qmk.cli.cformat', 'qmk.cli.chibios.confmigrate', 'qmk.cli.clean', 'qmk.cli.compile', - 'qmk.cli.console', 'qmk.cli.docs', 'qmk.cli.doctor', 'qmk.cli.fileformat', @@ -45,7 +45,9 @@ subcommands = [ 'qmk.cli.format.python', 'qmk.cli.format.text', 'qmk.cli.generate.api', + 'qmk.cli.generate.compilation_database', 'qmk.cli.generate.config_h', + 'qmk.cli.generate.develop_pr_list', 'qmk.cli.generate.dfu_header', 'qmk.cli.generate.docs', 'qmk.cli.generate.info_json', diff --git a/lib/python/qmk/cli/cd.py b/lib/python/qmk/cli/cd.py new file mode 100755 index 0000000000..c62c3f56c6 --- /dev/null +++ b/lib/python/qmk/cli/cd.py @@ -0,0 +1,46 @@ +"""Open a shell in the QMK Home directory +""" +import sys +import os + +from milc import cli + +from qmk.path import under_qmk_firmware + + +@cli.subcommand('Go to QMK Home') +def cd(cli): + """Go to QMK Home + """ + if not sys.stdout.isatty(): + cli.log.error("This command is for interactive usage only. For non-interactive usage, 'cd $(qmk env QMK_HOME)' is more robust.") + sys.exit(1) + + if not under_qmk_firmware(): + # Only do anything if the user is not under qmk_firmware already + # in order to reduce the possibility of starting multiple shells + cli.log.info("Spawning a subshell in your QMK_HOME directory.") + cli.log.info("Type 'exit' to get back to the parent shell.") + if not cli.platform.lower().startswith('windows'): + # For Linux/Mac/etc + # Check the user's login shell from 'passwd' + # alternatively fall back to $SHELL env var + # and finally to '/bin/bash'. + import getpass + import pwd + shell = pwd.getpwnam(getpass.getuser()).pw_shell + if not shell: + shell = os.environ.get('SHELL', '/bin/bash') + # Start the new subshell + os.execl(shell, shell) + else: + # For Windows + # Check the $SHELL env var + # and fall back to '/usr/bin/bash'. + qmk_env = os.environ.copy() + # Set the prompt for the new shell + qmk_env['MSYS2_PS1'] = qmk_env['PS1'] + # Start the new subshell + cli.run([os.environ.get('SHELL', '/usr/bin/bash')], env=qmk_env) + else: + cli.log.info("Already within qmk_firmware directory.") diff --git a/lib/python/qmk/cli/console.py b/lib/python/qmk/cli/console.py deleted file mode 100644 index 98c6bc0dc0..0000000000 --- a/lib/python/qmk/cli/console.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Acquire debugging information from usb hid devices - -cli implementation of https://www.pjrc.com/teensy/hid_listen.html -""" -from pathlib import Path -from threading import Thread -from time import sleep, strftime - -import hid -import usb.core - -from milc import cli - -LOG_COLOR = { - 'next': 0, - 'colors': [ - '{fg_blue}', - '{fg_cyan}', - '{fg_green}', - '{fg_magenta}', - '{fg_red}', - '{fg_yellow}', - ], -} - -KNOWN_BOOTLOADERS = { - # VID , PID - ('03EB', '2FEF'): 'atmel-dfu: ATmega16U2', - ('03EB', '2FF0'): 'atmel-dfu: ATmega32U2', - ('03EB', '2FF3'): 'atmel-dfu: ATmega16U4', - ('03EB', '2FF4'): 'atmel-dfu: ATmega32U4', - ('03EB', '2FF9'): 'atmel-dfu: AT90USB64', - ('03EB', '2FFA'): 'atmel-dfu: AT90USB162', - ('03EB', '2FFB'): 'atmel-dfu: AT90USB128', - ('03EB', '6124'): 'Microchip SAM-BA', - ('0483', 'DF11'): 'stm32-dfu: STM32 BOOTLOADER', - ('16C0', '05DC'): 'usbasploader: USBaspLoader', - ('16C0', '05DF'): 'bootloadhid: HIDBoot', - ('16C0', '0478'): 'halfkay: Teensy Halfkay', - ('1B4F', '9203'): 'caterina: Pro Micro 3.3V', - ('1B4F', '9205'): 'caterina: Pro Micro 5V', - ('1B4F', '9207'): 'caterina: LilyPadUSB', - ('1C11', 'B007'): 'kiibohd: Kiibohd DFU Bootloader', - ('1EAF', '0003'): 'stm32duino: Maple 003', - ('1FFB', '0101'): 'caterina: Polou A-Star 32U4 Bootloader', - ('2341', '0036'): 'caterina: Arduino Leonardo', - ('2341', '0037'): 'caterina: Arduino Micro', - ('239A', '000C'): 'caterina: Adafruit Feather 32U4', - ('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v', - ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v', - ('2A03', '0036'): 'caterina: Arduino Leonardo', - ('2A03', '0037'): 'caterina: Arduino Micro', - ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode', - ('03EB', '2067'): 'qmk-hid: HID Bootloader', - ('03EB', '2045'): 'lufa-ms: LUFA Mass Storage Bootloader' -} - - -class MonitorDevice(object): - def __init__(self, hid_device, numeric): - self.hid_device = hid_device - self.numeric = numeric - self.device = hid.Device(path=hid_device['path']) - self.current_line = '' - - cli.log.info('Console Connected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', hid_device) - - def read(self, size, encoding='ascii', timeout=1): - """Read size bytes from the device. - """ - return self.device.read(size, timeout).decode(encoding) - - def read_line(self): - """Read from the device's console until we get a \n. - """ - while '\n' not in self.current_line: - self.current_line += self.read(32).replace('\x00', '') - - lines = self.current_line.split('\n', 1) - self.current_line = lines[1] - - return lines[0] - - def run_forever(self): - while True: - try: - message = {**self.hid_device, 'text': self.read_line()} - identifier = (int2hex(message['vendor_id']), int2hex(message['product_id'])) if self.numeric else (message['manufacturer_string'], message['product_string']) - message['identifier'] = ':'.join(identifier) - message['ts'] = '{style_dim}{fg_green}%s{style_reset_all} ' % (strftime(cli.config.general.datetime_fmt),) if cli.args.timestamp else '' - - cli.echo('%(ts)s%(color)s%(identifier)s:%(index)d{style_reset_all}: %(text)s' % message) - - except hid.HIDException: - break - - -class FindDevices(object): - def __init__(self, vid, pid, index, numeric): - self.vid = vid - self.pid = pid - self.index = index - self.numeric = numeric - - def run_forever(self): - """Process messages from our queue in a loop. - """ - live_devices = {} - live_bootloaders = {} - - while True: - try: - for device in list(live_devices): - if not live_devices[device]['thread'].is_alive(): - cli.log.info('Console Disconnected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', live_devices[device]) - del live_devices[device] - - for device in self.find_devices(): - if device['path'] not in live_devices: - device['color'] = LOG_COLOR['colors'][LOG_COLOR['next']] - LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors']) - live_devices[device['path']] = device - - try: - monitor = MonitorDevice(device, self.numeric) - device['thread'] = Thread(target=monitor.run_forever, daemon=True) - - device['thread'].start() - except Exception as e: - device['e'] = e - device['e_name'] = e.__class__.__name__ - cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device) - if cli.config.general.verbose: - cli.log.exception(e) - del live_devices[device['path']] - - if cli.args.bootloaders: - for device in self.find_bootloaders(): - if device.address in live_bootloaders: - live_bootloaders[device.address]._qmk_found = True - else: - name = KNOWN_BOOTLOADERS[(int2hex(device.idVendor), int2hex(device.idProduct))] - cli.log.info('Bootloader Connected: {style_bright}{fg_magenta}%s', name) - device._qmk_found = True - live_bootloaders[device.address] = device - - for device in list(live_bootloaders): - if live_bootloaders[device]._qmk_found: - live_bootloaders[device]._qmk_found = False - else: - name = KNOWN_BOOTLOADERS[(int2hex(live_bootloaders[device].idVendor), int2hex(live_bootloaders[device].idProduct))] - cli.log.info('Bootloader Disconnected: {style_bright}{fg_magenta}%s', name) - del live_bootloaders[device] - - sleep(.1) - - except KeyboardInterrupt: - break - - def is_bootloader(self, hid_device): - """Returns true if the device in question matches a known bootloader vid/pid. - """ - return (int2hex(hid_device.idVendor), int2hex(hid_device.idProduct)) in KNOWN_BOOTLOADERS - - def is_console_hid(self, hid_device): - """Returns true when the usage page indicates it's a teensy-style console. - """ - return hid_device['usage_page'] == 0xFF31 and hid_device['usage'] == 0x0074 - - def is_filtered_device(self, hid_device): - """Returns True if the device should be included in the list of available consoles. - """ - return int2hex(hid_device['vendor_id']) == self.vid and int2hex(hid_device['product_id']) == self.pid - - def find_devices_by_report(self, hid_devices): - """Returns a list of available teensy-style consoles by doing a brute-force search. - - Some versions of linux don't report usage and usage_page. In that case we fallback to reading the report (possibly inaccurately) ourselves. - """ - devices = [] - - for device in hid_devices: - path = device['path'].decode('utf-8') - - if path.startswith('/dev/hidraw'): - number = path[11:] - report = Path(f'/sys/class/hidraw/hidraw{number}/device/report_descriptor') - - if report.exists(): - rp = report.read_bytes() - - if rp[1] == 0x31 and rp[3] == 0x09: - devices.append(device) - - return devices - - def find_bootloaders(self): - """Returns a list of available bootloader devices. - """ - return list(filter(self.is_bootloader, usb.core.find(find_all=True))) - - def find_devices(self): - """Returns a list of available teensy-style consoles. - """ - hid_devices = hid.enumerate() - devices = list(filter(self.is_console_hid, hid_devices)) - - if not devices: - devices = self.find_devices_by_report(hid_devices) - - if self.vid and self.pid: - devices = list(filter(self.is_filtered_device, devices)) - - # Add index numbers - device_index = {} - for device in devices: - id = ':'.join((int2hex(device['vendor_id']), int2hex(device['product_id']))) - - if id not in device_index: - device_index[id] = 0 - - device_index[id] += 1 - device['index'] = device_index[id] - - return devices - - -def int2hex(number): - """Returns a string representation of the number as hex. - """ - return "%04X" % number - - -def list_devices(device_finder): - """Show the user a nicely formatted list of devices. - """ - devices = device_finder.find_devices() - - if devices: - cli.log.info('Available devices:') - for dev in devices: - color = LOG_COLOR['colors'][LOG_COLOR['next']] - LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors']) - cli.log.info("\t%s%s:%s:%d{style_reset_all}\t%s %s", color, int2hex(dev['vendor_id']), int2hex(dev['product_id']), dev['index'], dev['manufacturer_string'], dev['product_string']) - - if cli.args.bootloaders: - bootloaders = device_finder.find_bootloaders() - - if bootloaders: - cli.log.info('Available Bootloaders:') - - for dev in bootloaders: - cli.log.info("\t%s:%s\t%s", int2hex(dev.idVendor), int2hex(dev.idProduct), KNOWN_BOOTLOADERS[(int2hex(dev.idVendor), int2hex(dev.idProduct))]) - - -@cli.argument('--bootloaders', arg_only=True, default=True, action='store_boolean', help='displaying bootloaders.') -@cli.argument('-d', '--device', help='Device to select - uses format <pid>:<vid>[:<index>].') -@cli.argument('-l', '--list', arg_only=True, action='store_true', help='List available hid_listen devices.') -@cli.argument('-n', '--numeric', arg_only=True, action='store_true', help='Show VID/PID instead of names.') -@cli.argument('-t', '--timestamp', arg_only=True, action='store_true', help='Print the timestamp for received messages as well.') -@cli.argument('-w', '--wait', type=int, default=1, help="How many seconds to wait between checks (Default: 1)") -@cli.subcommand('Acquire debugging information from usb hid devices.', hidden=False if cli.config.user.developer else True) -def console(cli): - """Acquire debugging information from usb hid devices - """ - vid = None - pid = None - index = 1 - - if cli.config.console.device: - device = cli.config.console.device.split(':') - - if len(device) == 2: - vid, pid = device - - elif len(device) == 3: - vid, pid, index = device - - if not index.isdigit(): - cli.log.error('Device index must be a number! Got "%s" instead.', index) - exit(1) - - index = int(index) - - if index < 1: - cli.log.error('Device index must be greater than 0! Got %s', index) - exit(1) - - else: - cli.log.error('Invalid format for device, expected "<pid>:<vid>[:<index>]" but got "%s".', cli.config.console.device) - cli.print_help() - exit(1) - - vid = vid.upper() - pid = pid.upper() - - device_finder = FindDevices(vid, pid, index, cli.args.numeric) - - if cli.args.list: - return list_devices(device_finder) - - print('Looking for devices...', flush=True) - device_finder.run_forever() diff --git a/lib/python/qmk/cli/docs.py b/lib/python/qmk/cli/docs.py index d8f9b045a1..c24b914bc1 100644 --- a/lib/python/qmk/cli/docs.py +++ b/lib/python/qmk/cli/docs.py @@ -2,6 +2,7 @@ """ import http.server import os +import shutil import webbrowser from milc import cli @@ -11,20 +12,33 @@ from milc import cli @cli.argument('-b', '--browser', action='store_true', help='Open the docs in the default browser.') @cli.subcommand('Run a local webserver for QMK documentation.', hidden=False if cli.config.user.developer else True) def docs(cli): - """Spin up a local HTTPServer instance for the QMK docs. + """Spin up a local HTTP server for the QMK docs. """ os.chdir('docs') - with http.server.HTTPServer(('', cli.config.docs.port), http.server.SimpleHTTPRequestHandler) as httpd: - cli.log.info(f"Serving QMK docs at http://localhost:{cli.config.docs.port}/") - cli.log.info("Press Control+C to exit.") + # If docsify-cli is installed, run that instead so we get live reload + if shutil.which('docsify'): + command = ['docsify', 'serve', '--port', f'{cli.config.docs.port}', '--open' if cli.config.docs.browser else ''] - if cli.config.docs.browser: - webbrowser.open(f'http://localhost:{cli.config.docs.port}') + cli.log.info(f"Running {{fg_cyan}}{str.join(' ', command)}{{fg_reset}}") + cli.log.info("Press Control+C to exit.") try: - httpd.serve_forever() + cli.run(command, capture_output=False) except KeyboardInterrupt: cli.log.info("Stopping HTTP server...") - finally: - httpd.shutdown() + else: + # Fall back to Python HTTPServer + with http.server.HTTPServer(('', cli.config.docs.port), http.server.SimpleHTTPRequestHandler) as httpd: + cli.log.info(f"Serving QMK docs at http://localhost:{cli.config.docs.port}/") + cli.log.info("Press Control+C to exit.") + + if cli.config.docs.browser: + webbrowser.open(f'http://localhost:{cli.config.docs.port}') + + try: + httpd.serve_forever() + except KeyboardInterrupt: + cli.log.info("Stopping HTTP server...") + finally: + httpd.shutdown() diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py index 0807f41518..2d691b64b0 100644 --- a/lib/python/qmk/cli/doctor/check.py +++ b/lib/python/qmk/cli/doctor/check.py @@ -26,7 +26,6 @@ ESSENTIAL_BINARIES = { 'arm-none-eabi-gcc': { 'version_arg': '-dumpversion' }, - 'bin/qmk': {}, } diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index c2d9e09c69..28e48a4101 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -18,17 +18,21 @@ def print_bootloader_help(): """Prints the available bootloaders listed in docs.qmk.fm. """ cli.log.info('Here are the available bootloaders:') + cli.echo('\tavrdude') + cli.echo('\tbootloadhid') cli.echo('\tdfu') + cli.echo('\tdfu-util') + cli.echo('\tmdloader') + cli.echo('\tst-flash') + cli.echo('\tst-link-cli') + cli.log.info('Enhanced variants for split keyboards:') + cli.echo('\tavrdude-split-left') + cli.echo('\tavrdude-split-right') cli.echo('\tdfu-ee') cli.echo('\tdfu-split-left') cli.echo('\tdfu-split-right') - cli.echo('\tavrdude') - cli.echo('\tBootloadHID') - cli.echo('\tdfu-util') cli.echo('\tdfu-util-split-left') cli.echo('\tdfu-util-split-right') - cli.echo('\tst-link-cli') - cli.echo('\tst-flash') cli.echo('For more info, visit https://docs.qmk.fm/#/flashing') diff --git a/lib/python/qmk/cli/format/c.py b/lib/python/qmk/cli/format/c.py index 0160e6036f..568684ed56 100644 --- a/lib/python/qmk/cli/format/c.py +++ b/lib/python/qmk/cli/format/c.py @@ -74,7 +74,7 @@ def filter_files(files, core_only=False): # The following statement checks each file to see if the file path is # - in the core directories # - not in the ignored directories - if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored): + if not any(str(file).startswith(i) for i in core_dirs) or any(str(file).startswith(i) for i in ignored): files[index] = None cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file) diff --git a/lib/python/qmk/cli/format/python.py b/lib/python/qmk/cli/format/python.py index 00612f97ec..47b5c45fd5 100755 --- a/lib/python/qmk/cli/format/python.py +++ b/lib/python/qmk/cli/format/python.py @@ -4,23 +4,66 @@ from subprocess import CalledProcessError, DEVNULL from milc import cli +from qmk.path import normpath -@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") -@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) -def format_python(cli): - """Format python code according to QMK's style. - """ +py_file_suffixes = ('py',) +py_dirs = ['lib/python'] + + +def yapf_run(files): edit = '--diff' if cli.args.dry_run else '--in-place' - yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] + yapf_cmd = ['yapf', '-vv', '--recursive', edit, *files] try: cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) - cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') - return True + cli.log.info('Successfully formatted the python code.') except CalledProcessError: - if cli.args.dry_run: - cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') + cli.log.error(f'Python code in {",".join(py_dirs)} incorrectly formatted!') + return False + + +def filter_files(files): + """Yield only files to be formatted and skip the rest + """ + for file in files: + if file and normpath(file).name.split('.')[-1] in py_file_suffixes: + yield file else: - cli.log.error('Error formatting python code!') + cli.log.debug('Skipping file %s', file) + + +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") +@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') +@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all files.') +@cli.argument('files', nargs='*', arg_only=True, type=normpath, help='Filename(s) to format.') +@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) +def format_python(cli): + """Format python code according to QMK's style. + """ + # Find the list of files to format + if cli.args.files: + files = list(filter_files(cli.args.files)) + + if not files: + cli.log.error('No Python files in filelist: %s', ', '.join(map(str, cli.args.files))) + exit(0) + + if cli.args.all_files: + cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) + + elif cli.args.all_files: + git_ls_cmd = ['git', 'ls-files', *py_dirs] + git_ls = cli.run(git_ls_cmd, stdin=DEVNULL) + files = list(filter_files(git_ls.stdout.split('\n'))) + + else: + git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *py_dirs] + git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) + files = list(filter_files(git_diff.stdout.split('\n'))) + + # Sanity check + if not files: + cli.log.error('No changed files detected. Use "qmk format-python -a" to format all files') + return False - return False + return yapf_run(files) diff --git a/lib/python/qmk/cli/format/text.py b/lib/python/qmk/cli/format/text.py index e7e07b7297..6dd4511896 100644 --- a/lib/python/qmk/cli/format/text.py +++ b/lib/python/qmk/cli/format/text.py @@ -1,27 +1,57 @@ """Ensure text files have the proper line endings. """ -from subprocess import CalledProcessError +from itertools import islice +from subprocess import DEVNULL from milc import cli +from qmk.path import normpath + +def _get_chunks(it, size): + """Break down a collection into smaller parts + """ + it = iter(it) + return iter(lambda: tuple(islice(it, size)), ()) + + +def dos2unix_run(files): + """Spawn multiple dos2unix subprocess avoiding too long commands on formatting everything + """ + for chunk in _get_chunks(files, 10): + dos2unix = cli.run(['dos2unix', *chunk]) + + if dos2unix.returncode: + return False + + +@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') +@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all files.') +@cli.argument('files', nargs='*', arg_only=True, type=normpath, help='Filename(s) to format.') @cli.subcommand("Ensure text files have the proper line endings.", hidden=True) def format_text(cli): """Ensure text files have the proper line endings. """ - try: - file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True) - except CalledProcessError as e: - cli.log.error('Could not get file list: %s', e) - exit(1) - except Exception as e: - cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e) - cli.log.exception(e) - exit(1) - - dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout) - - if dos2unix.returncode != 0: - print(dos2unix.stderr) - - return dos2unix.returncode + # Find the list of files to format + if cli.args.files: + files = list(cli.args.files) + + if cli.args.all_files: + cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files))) + + elif cli.args.all_files: + git_ls_cmd = ['git', 'ls-files'] + git_ls = cli.run(git_ls_cmd, stdin=DEVNULL) + files = list(filter(None, git_ls.stdout.split('\n'))) + + else: + git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch] + git_diff = cli.run(git_diff_cmd, stdin=DEVNULL) + files = list(filter(None, git_diff.stdout.split('\n'))) + + # Sanity check + if not files: + cli.log.error('No changed files detected. Use "qmk format-text -a" to format all files') + return False + + return dos2unix_run(files) diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py new file mode 100755 index 0000000000..602635270c --- /dev/null +++ b/lib/python/qmk/cli/generate/compilation_database.py @@ -0,0 +1,133 @@ +"""Creates a compilation database for the given keyboard build. +""" + +import json +import os +import re +import shlex +import shutil +from functools import lru_cache +from pathlib import Path +from typing import Dict, Iterator, List, Union + +from milc import cli, MILC + +from qmk.commands import create_make_command +from qmk.constants import QMK_FIRMWARE +from qmk.decorators import automagic_keyboard, automagic_keymap + + +@lru_cache(maxsize=10) +def system_libs(binary: str) -> List[Path]: + """Find the system include directory that the given build tool uses. + """ + cli.log.debug("searching for system library directory for binary: %s", binary) + bin_path = shutil.which(binary) + + # Actually query xxxxxx-gcc to find its include paths. + if binary.endswith("gcc") or binary.endswith("g++"): + result = cli.run([binary, '-E', '-Wp,-v', '-'], capture_output=True, check=True, input='\n') + paths = [] + for line in result.stderr.splitlines(): + if line.startswith(" "): + paths.append(Path(line.strip()).resolve()) + return paths + + return list(Path(bin_path).resolve().parent.parent.glob("*/include")) if bin_path else [] + + +file_re = re.compile(r'printf "Compiling: ([^"]+)') +cmd_re = re.compile(r'LOG=\$\((.+?)&&') + + +def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]: + """parse the output of `make -n <target>` + + This function makes many assumptions about the format of your build log. + This happens to work right now for qmk. + """ + + state = 'start' + this_file = None + records = [] + for line in f: + if state == 'start': + m = file_re.search(line) + if m: + this_file = m.group(1) + state = 'cmd' + + if state == 'cmd': + assert this_file + m = cmd_re.search(line) + if m: + # we have a hit! + this_cmd = m.group(1) + args = shlex.split(this_cmd) + for s in system_libs(args[0]): + args += ['-isystem', '%s' % s] + new_cmd = ' '.join(shlex.quote(s) for s in args if s != '-mno-thumb-interwork') + records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) + state = 'start' + + return records + + +@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.subcommand('Create a compilation database.') +@automagic_keyboard +@automagic_keymap +def generate_compilation_database(cli: MILC) -> Union[bool, int]: + """Creates a compilation database for the given keyboard build. + + Does a make clean, then a make -n for this target and uses the dry-run output to create + a compilation database (compile_commands.json). This file can help some IDEs and + IDE-like editors work better. For more information about this: + + https://clang.llvm.org/docs/JSONCompilationDatabase.html + """ + command = None + # check both config domains: the magic decorator fills in `generate_compilation_database` but the user is + # more likely to have set `compile` in their config file. + current_keyboard = cli.config.generate_compilation_database.keyboard or cli.config.user.keyboard + current_keymap = cli.config.generate_compilation_database.keymap or cli.config.user.keymap + + if current_keyboard and current_keymap: + # Generate the make command for a specific keyboard/keymap. + command = create_make_command(current_keyboard, current_keymap, dry_run=True) + elif not current_keyboard: + cli.log.error('Could not determine keyboard!') + elif not current_keymap: + cli.log.error('Could not determine keymap!') + + if not command: + cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.echo('usage: qmk compiledb [-kb KEYBOARD] [-km KEYMAP]') + return False + + # remove any environment variable overrides which could trip us up + env = os.environ.copy() + env.pop("MAKEFLAGS", None) + + # re-use same executable as the main make invocation (might be gmake) + clean_command = [command[0], 'clean'] + cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) + cli.run(clean_command, capture_output=False, check=True, env=env) + + cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) + + result = cli.run(command, capture_output=True, check=True, env=env) + db = parse_make_n(result.stdout.splitlines()) + if not db: + cli.log.error("Failed to parse output from make output:\n%s", result.stdout) + return False + + cli.log.info("Found %s compile commands", len(db)) + + dbpath = QMK_FIRMWARE / 'compile_commands.json' + + cli.log.info(f"Writing build database to {dbpath}") + dbpath.write_text(json.dumps(db, indent=4)) + + return True diff --git a/lib/python/qmk/cli/generate/develop_pr_list.py b/lib/python/qmk/cli/generate/develop_pr_list.py new file mode 100755 index 0000000000..07e46752a6 --- /dev/null +++ b/lib/python/qmk/cli/generate/develop_pr_list.py @@ -0,0 +1,119 @@ +"""Export the initial list of PRs associated with a `develop` merge to `master`. +""" +import os +import re +from pathlib import Path +from subprocess import DEVNULL + +from milc import cli + +cache_timeout = 7 * 86400 +fix_expr = re.compile(r'fix', flags=re.IGNORECASE) +clean1_expr = re.compile(r'\[(develop|keyboard|keymap|core|cli|bug|docs|feature)\]', flags=re.IGNORECASE) +clean2_expr = re.compile(r'^(develop|keyboard|keymap|core|cli|bug|docs|feature):', flags=re.IGNORECASE) + + +def _get_pr_info(cache, gh, pr_num): + pull = cache.get(f'pull:{pr_num}') + if pull is None: + print(f'Retrieving info for PR #{pr_num}') + pull = gh.pulls.get(owner='qmk', repo='qmk_firmware', pull_number=pr_num) + cache.set(f'pull:{pr_num}', pull, cache_timeout) + return pull + + +def _try_open_cache(cli): + # These dependencies are manually handled because people complain. Fun. + try: + from sqlite_cache.sqlite_cache import SqliteCache + except ImportError: + return None + + cache_loc = Path(cli.config_file).parent + return SqliteCache(cache_loc) + + +def _get_github(): + try: + from ghapi.all import GhApi + except ImportError: + return None + + return GhApi() + + +@cli.argument('-f', '--from-ref', default='0.11.0', help='Git revision/tag/reference/branch to begin search') +@cli.argument('-b', '--branch', default='upstream/develop', help='Git branch to iterate (default: "upstream/develop")') +@cli.subcommand('Creates the develop PR list.', hidden=False if cli.config.user.developer else True) +def generate_develop_pr_list(cli): + """Retrieves information from GitHub regarding the list of PRs associated + with a merge of `develop` branch into `master`. + + Requires environment variable GITHUB_TOKEN to be set. + """ + + if 'GITHUB_TOKEN' not in os.environ or os.environ['GITHUB_TOKEN'] == '': + cli.log.error('Environment variable "GITHUB_TOKEN" is not set.') + return 1 + + cache = _try_open_cache(cli) + gh = _get_github() + + git_args = ['git', 'rev-list', '--oneline', '--no-merges', '--reverse', f'{cli.args.from_ref}...{cli.args.branch}', '^upstream/master'] + commit_list = cli.run(git_args, capture_output=True, stdin=DEVNULL) + + if cache is None or gh is None: + cli.log.error('Missing one or more dependent python packages: "ghapi", "python-sqlite-cache"') + return 1 + + pr_list_bugs = [] + pr_list_dependencies = [] + pr_list_core = [] + pr_list_keyboards = [] + pr_list_keyboard_fixes = [] + pr_list_cli = [] + pr_list_others = [] + + def _categorise_commit(commit_info): + def fix_or_normal(info, fixes_collection, normal_collection): + if "bug" in info['pr_labels'] or fix_expr.search(info['title']): + fixes_collection.append(info) + else: + normal_collection.append(info) + + if "dependencies" in commit_info['pr_labels']: + fix_or_normal(commit_info, pr_list_bugs, pr_list_dependencies) + elif "core" in commit_info['pr_labels']: + fix_or_normal(commit_info, pr_list_bugs, pr_list_core) + elif "keyboard" in commit_info['pr_labels'] or "keymap" in commit_info['pr_labels'] or "via" in commit_info['pr_labels']: + fix_or_normal(commit_info, pr_list_keyboard_fixes, pr_list_keyboards) + elif "cli" in commit_info['pr_labels']: + fix_or_normal(commit_info, pr_list_bugs, pr_list_cli) + else: + fix_or_normal(commit_info, pr_list_bugs, pr_list_others) + + git_expr = re.compile(r'^(?P<hash>[a-f0-9]+) (?P<title>.*) \(#(?P<pr>[0-9]+)\)$') + for line in commit_list.stdout.split('\n'): + match = git_expr.search(line) + if match: + pr_info = _get_pr_info(cache, gh, match.group("pr")) + commit_info = {'hash': match.group("hash"), 'title': match.group("title"), 'pr_num': int(match.group("pr")), 'pr_labels': [label.name for label in pr_info.labels.items]} + _categorise_commit(commit_info) + + def _dump_commit_list(name, collection): + if len(collection) == 0: + return + print("") + print(f"{name}:") + for commit in sorted(collection, key=lambda x: x['pr_num']): + title = clean1_expr.sub('', clean2_expr.sub('', commit['title'])).strip() + pr_num = commit['pr_num'] + print(f'* {title} ([#{pr_num}](https://github.com/qmk/qmk_firmware/pull/{pr_num}))') + + _dump_commit_list("Core", pr_list_core) + _dump_commit_list("CLI", pr_list_cli) + _dump_commit_list("Submodule updates", pr_list_dependencies) + _dump_commit_list("Keyboards", pr_list_keyboards) + _dump_commit_list("Keyboard fixes", pr_list_keyboard_fixes) + _dump_commit_list("Others", pr_list_others) + _dump_commit_list("Bugs", pr_list_bugs) diff --git a/lib/python/qmk/cli/generate/dfu_header.py b/lib/python/qmk/cli/generate/dfu_header.py index 5a1b109f1e..7fb585fc7d 100644 --- a/lib/python/qmk/cli/generate/dfu_header.py +++ b/lib/python/qmk/cli/generate/dfu_header.py @@ -32,7 +32,7 @@ def generate_dfu_header(cli): keyboard_h_lines = ['/* This file was generated by `qmk generate-dfu-header`. Do not edit or copy.', ' */', '', '#pragma once'] keyboard_h_lines.append(f'#define MANUFACTURER {kb_info_json["manufacturer"]}') - keyboard_h_lines.append(f'#define PRODUCT {cli.config.generate_dfu_header.keyboard} Bootloader') + keyboard_h_lines.append(f'#define PRODUCT {kb_info_json["keyboard_name"]} Bootloader') # Optional if 'qmk_lufa_bootloader.esc_output' in kb_info_json: diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index dcaff29fae..5d8d7cc8a7 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -67,12 +67,9 @@ def generate_rules_mk(cli): # Iterate through features to enable/disable them if 'features' in kb_info_json: for feature, enabled in kb_info_json['features'].items(): - if feature == 'bootmagic_lite' and enabled: - rules_mk_lines.append('BOOTMAGIC_ENABLE ?= lite') - else: - feature = feature.upper() - enabled = 'yes' if enabled else 'no' - rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') + feature = feature.upper() + enabled = 'yes' if enabled else 'no' + rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') # Set SPLIT_TRANSPORT, if needed if kb_info_json.get('split', {}).get('transport', {}).get('protocol') == 'custom': diff --git a/lib/python/qmk/cli/json2c.py b/lib/python/qmk/cli/json2c.py index a90578c021..ae8248e6b7 100755 --- a/lib/python/qmk/cli/json2c.py +++ b/lib/python/qmk/cli/json2c.py @@ -33,7 +33,7 @@ def json2c(cli): cli.args.output = None # Generate the keymap - keymap_c = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers']) + keymap_c = qmk.keymap.generate_c(user_keymap) if cli.args.output: cli.args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/lib/python/qmk/cli/lint.py b/lib/python/qmk/cli/lint.py index 02b31fbc41..96593ed69b 100644 --- a/lib/python/qmk/cli/lint.py +++ b/lib/python/qmk/cli/lint.py @@ -1,72 +1,129 @@ """Command to look over a keyboard/keymap and check for common mistakes. """ +from pathlib import Path + from milc import cli from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.info import info_json -from qmk.keyboard import find_readme, keyboard_completer +from qmk.keyboard import keyboard_completer, list_keyboards from qmk.keymap import locate_keymap from qmk.path import is_keyboard, keyboard -@cli.argument('--strict', action='store_true', help='Treat warnings as errors.') -@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='The keyboard to check.') -@cli.argument('-km', '--keymap', help='The keymap to check.') +def keymap_check(kb, km): + """Perform the keymap level checks. + """ + ok = True + keymap_path = locate_keymap(kb, km) + + if not keymap_path: + ok = False + cli.log.error("%s: Can't find %s keymap.", kb, km) + + return ok + + +def rules_mk_assignment_only(keyboard_path): + """Check the keyboard-level rules.mk to ensure it only has assignments. + """ + current_path = Path() + errors = [] + + for path_part in keyboard_path.parts: + current_path = current_path / path_part + rules_mk = current_path / 'rules.mk' + + if rules_mk.exists(): + continuation = None + + for i, line in enumerate(rules_mk.open()): + line = line.strip() + + if '#' in line: + line = line[:line.index('#')] + + if continuation: + line = continuation + line + continuation = None + + if line: + if line[-1] == '\\': + continuation = line[:-1] + continue + + if line and '=' not in line: + errors.append(f'Non-assignment code on line +{i} {rules_mk}: {line}') + + return errors + + +@cli.argument('--strict', action='store_true', help='Treat warnings as errors') +@cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='Comma separated list of keyboards to check') +@cli.argument('-km', '--keymap', help='The keymap to check') +@cli.argument('--all-kb', action='store_true', arg_only=True, help='Check all keyboards') @cli.subcommand('Check keyboard and keymap for common mistakes.') @automagic_keyboard @automagic_keymap def lint(cli): """Check keyboard and keymap for common mistakes. """ - if not cli.config.lint.keyboard: - cli.log.error('Missing required argument: --keyboard') - cli.print_help() - return False + failed = [] - if not is_keyboard(cli.config.lint.keyboard): - cli.log.error('No such keyboard: %s', cli.config.lint.keyboard) - return False + # Determine our keyboard list + if cli.args.all_kb: + if cli.args.keyboard: + cli.log.warning('Both --all-kb and --keyboard passed, --all-kb takes presidence.') - # Gather data about the keyboard. - ok = True - keyboard_path = keyboard(cli.config.lint.keyboard) - keyboard_info = info_json(cli.config.lint.keyboard) - readme_path = find_readme(cli.config.lint.keyboard) - missing_readme_path = keyboard_path / 'readme.md' + keyboard_list = list_keyboards() + elif not cli.config.lint.keyboard: + cli.log.error('Missing required arguments: --keyboard or --all-kb') + cli.print_help() + return False + else: + keyboard_list = cli.config.lint.keyboard.split(',') - # Check for errors in the info.json - if keyboard_info['parse_errors']: - ok = False - cli.log.error('Errors found when generating info.json.') + # Lint each keyboard + for kb in keyboard_list: + if not is_keyboard(kb): + cli.log.error('No such keyboard: %s', kb) + continue - if cli.config.lint.strict and keyboard_info['parse_warnings']: - ok = False - cli.log.error('Warnings found when generating info.json (Strict mode enabled.)') + # Gather data about the keyboard. + ok = True + keyboard_path = keyboard(kb) + keyboard_info = info_json(kb) - # Check for a readme.md and warn if it doesn't exist - if not readme_path: - ok = False - cli.log.error('Missing %s', missing_readme_path) + # Check for errors in the info.json + if keyboard_info['parse_errors']: + ok = False + cli.log.error('%s: Errors found when generating info.json.', kb) - # Keymap specific checks - if cli.config.lint.keymap: - keymap_path = locate_keymap(cli.config.lint.keyboard, cli.config.lint.keymap) + if cli.config.lint.strict and keyboard_info['parse_warnings']: + ok = False + cli.log.error('%s: Warnings found when generating info.json (Strict mode enabled.)', kb) - if not keymap_path: + # Check the rules.mk file(s) + rules_mk_assignment_errors = rules_mk_assignment_only(keyboard_path) + if rules_mk_assignment_errors: ok = False - cli.log.error("Can't find %s keymap for %s keyboard.", cli.config.lint.keymap, cli.config.lint.keyboard) - else: - keymap_readme = keymap_path.parent / 'readme.md' - if not keymap_readme.exists(): - cli.log.warning('Missing %s', keymap_readme) + cli.log.error('%s: Non-assignment code found in rules.mk. Move it to post_rules.mk instead.', kb) + for assignment_error in rules_mk_assignment_errors: + cli.log.error(assignment_error) - if cli.config.lint.strict: - ok = False + # Keymap specific checks + if cli.config.lint.keymap: + if not keymap_check(kb, cli.config.lint.keymap): + ok = False + + # Report status + if not ok: + failed.append(kb) # Check and report the overall status - if ok: - cli.log.info('Lint check passed!') - return True + if failed: + cli.log.error('Lint check failed for: %s', ', '.join(failed)) + return False - cli.log.error('Lint check failed!') - return False + cli.log.info('Lint check passed!') + return True diff --git a/lib/python/qmk/cli/list/keymaps.py b/lib/python/qmk/cli/list/keymaps.py index d79ab75b58..d2ef136c06 100644 --- a/lib/python/qmk/cli/list/keymaps.py +++ b/lib/python/qmk/cli/list/keymaps.py @@ -13,5 +13,10 @@ from qmk.keyboard import keyboard_completer, keyboard_folder def list_keymaps(cli): """List the keymaps for a specific keyboard """ + if not cli.config.list_keymaps.keyboard: + cli.log.error('Missing required arguments: --keyboard') + cli.subcommands['list-keymaps'].print_help() + return False + for name in qmk.keymap.list_keymaps(cli.config.list_keymaps.keyboard): print(name) diff --git a/lib/python/qmk/cli/list/layouts.py b/lib/python/qmk/cli/list/layouts.py index 8e07afeeca..df593dc390 100644 --- a/lib/python/qmk/cli/list/layouts.py +++ b/lib/python/qmk/cli/list/layouts.py @@ -13,6 +13,11 @@ from qmk.info import info_json def list_layouts(cli): """List the layouts for a specific keyboard """ + if not cli.config.list_layouts.keyboard: + cli.log.error('Missing required arguments: --keyboard') + cli.subcommands['list-layouts'].print_help() + return False + info_data = info_json(cli.config.list_layouts.keyboard) for name in sorted(info_data.get('community_layouts', [])): print(name) diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index 369d2bd7da..4093b8c90d 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -1,10 +1,8 @@ """This script automates the creation of new keyboard directories using a starter template. """ from datetime import date -import fileinput from pathlib import Path import re -import shutil from qmk.commands import git_get_username import qmk.path @@ -32,6 +30,7 @@ def validate_keyboard_name(name): @cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name) @cli.argument('-t', '--type', help='Specify the keyboard type', arg_only=True, choices=KEYBOARD_TYPES) @cli.argument('-u', '--username', help='Specify your username (default from Git config)', arg_only=True) +@cli.argument('-n', '--realname', help='Specify your real name if you want to use that. Defaults to username', arg_only=True) @cli.subcommand('Creates a new keyboard directory') def new_keyboard(cli): """Creates a new keyboard. @@ -69,7 +68,7 @@ def new_keyboard(cli): # Get username user_name = None while not user_name: - user_name = question('Your Name:', default=find_user_name()) + user_name = question('Your GitHub User Name:', default=find_user_name()) if not user_name: cli.log.error('You didn\'t provide a username, and we couldn\'t find one set in your QMK or Git configs. Please try again.') @@ -78,26 +77,21 @@ def new_keyboard(cli): if cli.args.username: return False - # Copy all the files - copy_templates(keyboard_type, keyboard_path) + real_name = None + while not real_name: + real_name = question('Your real name:', default=user_name) - # Replace all the placeholders keyboard_basename = keyboard_path.name - replacements = [ - ('%YEAR%', str(date.today().year)), - ('%KEYBOARD%', keyboard_basename), - ('%YOUR_NAME%', user_name), - ] - filenames = [ - keyboard_path / 'config.h', - keyboard_path / 'info.json', - keyboard_path / 'readme.md', - keyboard_path / f'{keyboard_basename}.c', - keyboard_path / f'{keyboard_basename}.h', - keyboard_path / 'keymaps/default/readme.md', - keyboard_path / 'keymaps/default/keymap.c', - ] - replace_placeholders(replacements, filenames) + replacements = { + "YEAR": str(date.today().year), + "KEYBOARD": keyboard_basename, + "USER_NAME": user_name, + "YOUR_NAME": real_name, + } + + template_dir = Path('data/templates') + template_tree(template_dir / 'base', keyboard_path, replacements) + template_tree(template_dir / keyboard_type, keyboard_path, replacements) cli.echo('') cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{new_keyboard_name}{{fg_green}}.{{fg_reset}}') @@ -114,29 +108,32 @@ def find_user_name(): return git_get_username() -def copy_templates(keyboard_type, keyboard_path): - """Copies the template files from data/templates to the new keyboard directory. - """ - template_base_path = Path('data/templates') - keyboard_basename = keyboard_path.name +def template_tree(src: Path, dst: Path, replacements: dict): + """Recursively copy template and replace placeholders - cli.log.info('Copying base template files...') - shutil.copytree(template_base_path / 'base', keyboard_path) + Args: + src (Path) + The source folder to copy from + dst (Path) + The destination folder to copy to + replacements (dict) + a dictionary with "key":"value" pairs to replace. - cli.log.info(f'Copying {{fg_cyan}}{keyboard_type}{{fg_reset}} template files...') - shutil.copytree(template_base_path / keyboard_type, keyboard_path, dirs_exist_ok=True) + Raises: + FileExistsError + When trying to overwrite existing files + """ - cli.log.info(f'Renaming {{fg_cyan}}keyboard.[ch]{{fg_reset}} to {{fg_cyan}}{keyboard_basename}.[ch]{{fg_reset}}...') - shutil.move(keyboard_path / 'keyboard.c', keyboard_path / f'{keyboard_basename}.c') - shutil.move(keyboard_path / 'keyboard.h', keyboard_path / f'{keyboard_basename}.h') + dst.mkdir(parents=True, exist_ok=True) + for child in src.iterdir(): + if child.is_dir(): + template_tree(child, dst / child.name, replacements=replacements) -def replace_placeholders(replacements, filenames): - """Replaces the given placeholders in each template file. - """ - for replacement in replacements: - cli.log.info(f'Replacing {{fg_cyan}}{replacement[0]}{{fg_reset}} with {{fg_cyan}}{replacement[1]}{{fg_reset}}...') + if child.is_file(): + file_name = dst / (child.name % replacements) - with fileinput.input(files=filenames, inplace=True) as file: - for line in file: - print(line.replace(replacement[0], replacement[1]), end='') + with file_name.open(mode='x') as dst_f: + with child.open() as src_f: + template = src_f.read() + dst_f.write(template % replacements) diff --git a/lib/python/qmk/cli/pytest.py b/lib/python/qmk/cli/pytest.py index bdb336b9a7..a7f01a872a 100644 --- a/lib/python/qmk/cli/pytest.py +++ b/lib/python/qmk/cli/pytest.py @@ -12,6 +12,6 @@ def pytest(cli): """Run several linting/testing commands. """ nose2 = cli.run(['nose2', '-v'], capture_output=False, stdin=DEVNULL) - flake8 = cli.run(['flake8', 'lib/python', 'bin/qmk'], capture_output=False, stdin=DEVNULL) + flake8 = cli.run(['flake8', 'lib/python'], capture_output=False, stdin=DEVNULL) return flake8.returncode | nose2.returncode diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 421453d837..5a01943773 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -28,7 +28,7 @@ def _find_make(): return make_cmd -def create_make_target(target, parallel=1, **env_vars): +def create_make_target(target, dry_run=False, parallel=1, **env_vars): """Create a make command Args: @@ -36,6 +36,9 @@ def create_make_target(target, parallel=1, **env_vars): target Usually a make rule, such as 'clean' or 'all'. + dry_run + make -n -- don't actually build + parallel The number of make jobs to run in parallel @@ -52,10 +55,10 @@ def create_make_target(target, parallel=1, **env_vars): for key, value in env_vars.items(): env.append(f'{key}={value}') - return [make_cmd, *get_make_parallel_args(parallel), *env, target] + return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target] -def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): +def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars): """Create a make compile command Args: @@ -69,6 +72,9 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): target Usually a bootloader. + dry_run + make -n -- don't actually build + parallel The number of make jobs to run in parallel @@ -84,7 +90,7 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): if target: make_args.append(target) - return create_make_target(':'.join(make_args), parallel, **env_vars) + return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars) def get_git_version(current_time, repo_dir='.', check_dir='.'): @@ -184,7 +190,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va target = f'{keyboard_filesafe}_{user_keymap["keymap"]}' keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}') keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}') - c_text = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers']) + c_text = qmk.keymap.generate_c(user_keymap) keymap_dir = keymap_output / 'src' keymap_c = keymap_dir / 'keymap.c' @@ -233,7 +239,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va f'VERBOSE={verbose}', f'COLOR={color}', 'SILENT=false', - f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}', + 'QMK_BIN="qmk"', ]) return make_command diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 71a6c91c77..754091a97e 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -13,7 +13,7 @@ QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' MAX_KEYBOARD_SUBFOLDERS = 5 # Supported processor types -CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66F18', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L433', 'STM32L443' +CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66FX1M0', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L433', 'STM32L443', 'GD32VF103', 'WB32F3G71' LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85' diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 350e5e2178..9a07fc842f 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -25,6 +25,13 @@ def _valid_community_layout(layout): return (Path('layouts/default') / layout).exists() +def _remove_newlines_from_labels(layouts): + for layout_name, layout_json in layouts.items(): + for key in layout_json['layout']: + if '\n' in key['label']: + key['label'] = key['label'].split('\n')[0] + + def info_json(keyboard): """Generate the info.json data for a specific keyboard. """ @@ -99,6 +106,9 @@ def info_json(keyboard): # Check that the reported matrix size is consistent with the actual matrix size _check_matrix(info_data) + # Remove newline characters from layout labels + _remove_newlines_from_labels(layouts) + return info_data @@ -112,11 +122,6 @@ def _extract_features(info_data, rules): if rules.get('BOOTMAGIC_ENABLE') == 'full': rules['BOOTMAGIC_ENABLE'] = 'on' - # Skip non-boolean features we haven't implemented special handling for - for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': - if rules.get(feature): - del rules[feature] - # Process the rest of the rules as booleans for key, value in rules.items(): if key.endswith('_ENABLE'): @@ -619,6 +624,8 @@ def arm_processor_rules(info_data, rules): if 'bootloader' not in info_data: if 'STM32' in info_data['processor']: info_data['bootloader'] = 'stm32-dfu' + elif 'WB32' in info_data['processor']: + info_data['bootloader'] = 'wb32-dfu' else: info_data['bootloader'] = 'unknown' diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 6eec49cfd1..00b5a78a5a 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -17,6 +17,7 @@ from qmk.errors import CppError # The `keymap.c` template to use when a keyboard doesn't have its own DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H +__INCLUDES__ /* THIS FILE WAS GENERATED! * @@ -27,6 +28,7 @@ DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { __KEYMAP_GOES_HERE__ }; + """ @@ -180,10 +182,11 @@ def generate_json(keymap, keyboard, layout, layers): return new_keymap -def generate_c(keyboard, layout, layers): - """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers. +def generate_c(keymap_json): + """Returns a `keymap.c`. + + `keymap_json` is a dictionary with the following keys: - Args: keyboard The name of the keyboard @@ -192,19 +195,89 @@ def generate_c(keyboard, layout, layers): layers An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode. + + macros + A sequence of strings containing macros to implement for this keyboard. """ - new_keymap = template_c(keyboard) + new_keymap = template_c(keymap_json['keyboard']) layer_txt = [] - for layer_num, layer in enumerate(layers): + + for layer_num, layer in enumerate(keymap_json['layers']): if layer_num != 0: layer_txt[-1] = layer_txt[-1] + ',' layer = map(_strip_any, layer) layer_keys = ', '.join(layer) - layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys)) + layer_txt.append('\t[%s] = %s(%s)' % (layer_num, keymap_json['layout'], layer_keys)) keymap = '\n'.join(layer_txt) new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap) + if keymap_json.get('macros'): + macro_txt = [ + 'bool process_record_user(uint16_t keycode, keyrecord_t *record) {', + ' if (record->event.pressed) {', + ' switch (keycode) {', + ] + + for i, macro_array in enumerate(keymap_json['macros']): + macro = [] + + for macro_fragment in macro_array: + if isinstance(macro_fragment, str): + macro_fragment = macro_fragment.replace('\\', '\\\\') + macro_fragment = macro_fragment.replace('\r\n', r'\n') + macro_fragment = macro_fragment.replace('\n', r'\n') + macro_fragment = macro_fragment.replace('\r', r'\n') + macro_fragment = macro_fragment.replace('\t', r'\t') + macro_fragment = macro_fragment.replace('"', r'\"') + + macro.append(f'"{macro_fragment}"') + + elif isinstance(macro_fragment, dict): + newstring = [] + + if macro_fragment['action'] == 'delay': + newstring.append(f"SS_DELAY({macro_fragment['duration']})") + + elif macro_fragment['action'] == 'beep': + newstring.append(r'"\a"') + + elif macro_fragment['action'] == 'tap' and len(macro_fragment['keycodes']) > 1: + last_keycode = macro_fragment['keycodes'].pop() + + for keycode in macro_fragment['keycodes']: + newstring.append(f'SS_DOWN(X_{keycode})') + + newstring.append(f'SS_TAP(X_{last_keycode})') + + for keycode in reversed(macro_fragment['keycodes']): + newstring.append(f'SS_UP(X_{keycode})') + + else: + for keycode in macro_fragment['keycodes']: + newstring.append(f"SS_{macro_fragment['action'].upper()}(X_{keycode})") + + macro.append(''.join(newstring)) + + new_macro = "".join(macro) + new_macro = new_macro.replace('""', '') + macro_txt.append(f' case MACRO_{i}:') + macro_txt.append(f' SEND_STRING({new_macro});') + macro_txt.append(' return false;') + + macro_txt.append(' }') + macro_txt.append(' }') + macro_txt.append('\n return true;') + macro_txt.append('};') + macro_txt.append('') + + new_keymap = '\n'.join((new_keymap, *macro_txt)) + + if keymap_json.get('host_language'): + new_keymap = new_keymap.replace('__INCLUDES__', f'#include "keymap_{keymap_json["host_language"]}.h"\n#include "sendstring_{keymap_json["host_language"]}.h"\n') + else: + new_keymap = new_keymap.replace('__INCLUDES__', '') + return new_keymap @@ -217,7 +290,7 @@ def write_file(keymap_filename, keymap_content): return keymap_filename -def write_json(keyboard, keymap, layout, layers): +def write_json(keyboard, keymap, layout, layers, macros=None): """Generate the `keymap.json` and write it to disk. Returns the filename written to. @@ -235,19 +308,19 @@ def write_json(keyboard, keymap, layout, layers): layers An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode. """ - keymap_json = generate_json(keyboard, keymap, layout, layers) + keymap_json = generate_json(keyboard, keymap, layout, layers, macros=None) keymap_content = json.dumps(keymap_json) keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json' return write_file(keymap_file, keymap_content) -def write(keyboard, keymap, layout, layers): +def write(keymap_json): """Generate the `keymap.c` and write it to disk. Returns the filename written to. - Args: + `keymap_json` should be a dict with the following keys: keyboard The name of the keyboard @@ -259,9 +332,12 @@ def write(keyboard, keymap, layout, layers): layers An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode. + + macros + A list of macros for this keymap. """ - keymap_content = generate_c(keyboard, layout, layers) - keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c' + keymap_content = generate_c(keymap_json) + keymap_file = qmk.path.keymap(keymap_json['keyboard']) / keymap_json['keymap'] / 'keymap.c' return write_file(keymap_file, keymap_content) diff --git a/lib/python/qmk/tests/minimal_info.json b/lib/python/qmk/tests/minimal_info.json index 11ef12fefe..3aae4722bf 100644 --- a/lib/python/qmk/tests/minimal_info.json +++ b/lib/python/qmk/tests/minimal_info.json @@ -4,7 +4,7 @@ "layouts": { "LAYOUT": { "layout": [ - { "label": "KC_A", "x": 0, "y": 0, "matrix": [0, 0] } + { "label": "KC_A", "matrix": [0, 0], "x": 0, "y": 0 } ] } } diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index b39fe5e46d..2973f81702 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -81,9 +81,9 @@ def test_hello(): def test_format_python(): - result = check_subcommand('format-python', '--dry-run') + result = check_subcommand('format-python', '-n', '-a') check_returncode(result) - assert 'Python code in `bin/qmk` and `lib/python` is correctly formatted.' in result.stdout + assert 'Successfully formatted the python code.' in result.stdout def test_list_keyboards(): @@ -142,6 +142,14 @@ def test_json2c(): assert result.stdout == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT_ortho_1x1(KC_A)};\n\n' +def test_json2c_macros(): + result = check_subcommand("json2c", 'keyboards/handwired/pytest/macro/keymaps/default/keymap.json') + check_returncode(result) + assert 'LAYOUT_ortho_1x1(MACRO_0)' in result.stdout + assert 'case MACRO_0:' in result.stdout + assert 'SEND_STRING("Hello, World!"SS_TAP(X_ENTER));' in result.stdout + + def test_json2c_stdin(): result = check_subcommand_stdin('keyboards/handwired/pytest/has_template/keymaps/default_json/keymap.json', 'json2c', '-') check_returncode(result) @@ -151,7 +159,7 @@ def test_json2c_stdin(): def test_info(): result = check_subcommand('info', '-kb', 'handwired/pytest/basic') check_returncode(result) - assert 'Keyboard Name: handwired/pytest/basic' in result.stdout + assert 'Keyboard Name: pytest' in result.stdout assert 'Processor: atmega32u4' in result.stdout assert 'Layout:' not in result.stdout assert 'k0' not in result.stdout @@ -160,7 +168,7 @@ def test_info(): def test_info_keyboard_render(): result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-l') check_returncode(result) - assert 'Keyboard Name: handwired/pytest/basic' in result.stdout + assert 'Keyboard Name: pytest' in result.stdout assert 'Processor: atmega32u4' in result.stdout assert 'Layouts:' in result.stdout assert 'k0' in result.stdout @@ -169,7 +177,7 @@ def test_info_keyboard_render(): def test_info_keymap_render(): result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-km', 'default_json') check_returncode(result) - assert 'Keyboard Name: handwired/pytest/basic' in result.stdout + assert 'Keyboard Name: pytest' in result.stdout assert 'Processor: atmega32u4' in result.stdout if is_windows: @@ -181,7 +189,7 @@ def test_info_keymap_render(): def test_info_matrix_render(): result = check_subcommand('info', '-kb', 'handwired/pytest/basic', '-m') check_returncode(result) - assert 'Keyboard Name: handwired/pytest/basic' in result.stdout + assert 'Keyboard Name: pytest' in result.stdout assert 'Processor: atmega32u4' in result.stdout assert 'LAYOUT_ortho_1x1' in result.stdout @@ -242,7 +250,7 @@ def test_generate_config_h(): assert '# define DESCRIPTION handwired/pytest/basic' in result.stdout assert '# define DIODE_DIRECTION COL2ROW' in result.stdout assert '# define MANUFACTURER none' in result.stdout - assert '# define PRODUCT handwired/pytest/basic' in result.stdout + assert '# define PRODUCT pytest' in result.stdout assert '# define PRODUCT_ID 0x6465' in result.stdout assert '# define VENDOR_ID 0xFEED' in result.stdout assert '# define MATRIX_COLS 1' in result.stdout diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py index b9e80df672..5e2efc1232 100644 --- a/lib/python/qmk/tests/test_qmk_keymap.py +++ b/lib/python/qmk/tests/test_qmk_keymap.py @@ -22,7 +22,13 @@ def test_template_json_pytest_has_template(): def test_generate_c_pytest_has_template(): - templ = qmk.keymap.generate_c('handwired/pytest/has_template', 'LAYOUT', [['KC_A']]) + keymap_json = { + 'keyboard': 'handwired/pytest/has_template', + 'layout': 'LAYOUT', + 'layers': [['KC_A']], + 'macros': None, + } + templ = qmk.keymap.generate_c(keymap_json) assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n' |