diff options
author | Erovia <Erovia@users.noreply.github.com> | 2022-08-20 06:39:19 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-20 15:39:19 +1000 |
commit | 5e2ffe7d8f4187109514147469d7db93e075f6f0 (patch) | |
tree | 83552a38351b50f6ab175c6abd6070abcc3d78a0 /lib/python | |
parent | 3bf36e8b04be5af9826fe3241d227efb185a4a31 (diff) |
CLI: Teaching the CLI to flash binaries (#16584)
Co-authored-by: Ryan <fauxpark@gmail.com>
Co-authored-by: Sergey Vlasov <sigprof@gmail.com>
Co-authored-by: Joel Challis <git@zvecr.com>
Co-authored-by: Nick Brassel <nick@tzarc.org>
Diffstat (limited to 'lib/python')
-rw-r--r-- | lib/python/qmk/cli/__init__.py | 1 | ||||
-rw-r--r-- | lib/python/qmk/cli/doctor/linux.py | 57 | ||||
-rw-r--r-- | lib/python/qmk/cli/flash.py | 120 | ||||
-rw-r--r-- | lib/python/qmk/constants.py | 48 | ||||
-rw-r--r-- | lib/python/qmk/flashers.py | 203 |
5 files changed, 336 insertions, 93 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 8a507677ef..f05b2a746e 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -15,6 +15,7 @@ from milc.questions import yesno import_names = { # A mapping of package name to importable name 'pep8-naming': 'pep8ext_naming', + 'pyserial': 'serial', 'pyusb': 'usb.core', 'qmk-dotty-dict': 'dotty_dict', 'pillow': 'PIL' diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py index 94683d3307..a803305c0d 100644 --- a/lib/python/qmk/cli/doctor/linux.py +++ b/lib/python/qmk/cli/doctor/linux.py @@ -6,7 +6,7 @@ from pathlib import Path from milc import cli -from qmk.constants import QMK_FIRMWARE +from qmk.constants import QMK_FIRMWARE, BOOTLOADER_VIDS_PIDS from .check import CheckStatus @@ -26,6 +26,18 @@ def _udev_rule(vid, pid=None, *args): return rule +def _generate_desired_rules(bootloader_vids_pids): + rules = dict() + for bl in bootloader_vids_pids.keys(): + rules[bl] = set() + for vid_pid in bootloader_vids_pids[bl]: + if bl == 'caterina' or bl == 'md-boot': + rules[bl].add(_udev_rule(vid_pid[0], vid_pid[1], 'ENV{ID_MM_DEVICE_IGNORE}="1"')) + else: + rules[bl].add(_udev_rule(vid_pid[0], vid_pid[1])) + return rules + + def _deprecated_udev_rule(vid, pid=None): """ Helper function that return udev rules @@ -47,47 +59,8 @@ def check_udev_rules(): Path("/run/udev/rules.d/"), Path("/etc/udev/rules.d/"), ] - desired_rules = { - 'atmel-dfu': { - _udev_rule("03eb", "2fef"), # ATmega16U2 - _udev_rule("03eb", "2ff0"), # ATmega32U2 - _udev_rule("03eb", "2ff3"), # ATmega16U4 - _udev_rule("03eb", "2ff4"), # ATmega32U4 - _udev_rule("03eb", "2ff9"), # AT90USB64 - _udev_rule("03eb", "2ffa"), # AT90USB162 - _udev_rule("03eb", "2ffb") # AT90USB128 - }, - 'kiibohd': {_udev_rule("1c11", "b007")}, - 'stm32': { - _udev_rule("1eaf", "0003"), # STM32duino - _udev_rule("0483", "df11") # STM32 DFU - }, - 'bootloadhid': {_udev_rule("16c0", "05df")}, - 'usbasploader': {_udev_rule("16c0", "05dc")}, - 'massdrop': {_udev_rule("03eb", "6124", 'ENV{ID_MM_DEVICE_IGNORE}="1"')}, - 'caterina': { - # Spark Fun Electronics - _udev_rule("1b4f", "9203", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 3V3/8MHz - _udev_rule("1b4f", "9205", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 5V/16MHz - _udev_rule("1b4f", "9207", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # LilyPad 3V3/8MHz (and some Pro Micro clones) - # Pololu Electronics - _udev_rule("1ffb", "0101", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # A-Star 32U4 - # Arduino SA - _udev_rule("2341", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo - _udev_rule("2341", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Micro - # Adafruit Industries LLC - _udev_rule("239a", "000c", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Feather 32U4 - _udev_rule("239a", "000d", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 3V3/8MHz - _udev_rule("239a", "000e", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 5V/16MHz - # dog hunter AG - _udev_rule("2a03", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo - _udev_rule("2a03", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"') # Micro - }, - 'hid-bootloader': { - _udev_rule("03eb", "2067"), # QMK HID - _udev_rule("16c0", "0478") # PJRC halfkay - } - } + + desired_rules = _generate_desired_rules(BOOTLOADER_VIDS_PIDS) # These rules are no longer recommended, only use them to check for their presence. deprecated_rules = { diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index ebe739c50e..c39f4b36d4 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -4,6 +4,7 @@ You can compile a keymap already in the repo or using a QMK Configurator export. A bootloader must be specified. """ from subprocess import DEVNULL +import sys from argcomplete.completers import FilesCompleter from milc import cli @@ -12,6 +13,7 @@ import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.flashers import flasher def print_bootloader_help(): @@ -38,9 +40,10 @@ def print_bootloader_help(): cli.echo('For more info, visit https://docs.qmk.fm/#/flashing') -@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export JSON to compile.') +@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.') @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.') @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.') +@cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @@ -53,6 +56,8 @@ def print_bootloader_help(): def flash(cli): """Compile and or flash QMK Firmware or keyboard/layout + If a binary firmware is supplied, try to flash that. + If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments will be ignored. @@ -60,56 +65,69 @@ def flash(cli): If bootloader is omitted the make system will use the configured bootloader for that keyboard. """ - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - if cli.config.flash.keyboard and cli.config.flash.keymap: - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) - - # Build the environment vars - envs = {} - for env in cli.args.env: - if '=' in env: - key, value = env.split('=', 1) - envs[key] = value - else: - cli.log.warning('Invalid environment variable: %s', env) - - # Determine the compile command - command = '' - - if cli.args.bootloaders: - # Provide usage and list bootloaders - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') - print_bootloader_help() - return False - - if cli.args.filename: - # Handle compiling a configurator JSON - user_keymap = parse_configurator_json(cli.args.filename) - keymap_path = qmk.path.keymap(user_keymap['keyboard']) - command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap']) + if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']: + # Try to flash binary firmware + cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n') + try: + err, msg = flasher(cli.args.mcu, cli.args.filename) + if err: + cli.log.error(msg) + return False + except KeyboardInterrupt: + cli.log.info('Ctrl-C was pressed, exiting...') + sys.exit(0) else: - if cli.config.flash.keyboard and cli.config.flash.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - elif not cli.config.flash.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.flash.keymap: - cli.log.error('Could not determine keymap!') - - # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - compile = cli.run(command, capture_output=False, stdin=DEVNULL) - return compile.returncode + if cli.args.clean and not cli.args.filename and not cli.args.dry_run: + if cli.config.flash.keyboard and cli.config.flash.keymap: + command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean') + cli.run(command, capture_output=False, stdin=DEVNULL) + + # Build the environment vars + envs = {} + for env in cli.args.env: + if '=' in env: + key, value = env.split('=', 1) + envs[key] = value + else: + cli.log.warning('Invalid environment variable: %s', env) + + # Determine the compile command + command = '' + + if cli.args.bootloaders: + # Provide usage and list bootloaders + cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') + print_bootloader_help() + return False + + if cli.args.filename: + # Handle compiling a configurator JSON + user_keymap = parse_configurator_json(cli.args.filename) + keymap_path = qmk.path.keymap(user_keymap['keyboard']) + command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) + + cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap']) - else: - cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') - return False + else: + if cli.config.flash.keyboard and cli.config.flash.keymap: + # Generate the make command for a specific keyboard/keymap. + command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) + + elif not cli.config.flash.keyboard: + cli.log.error('Could not determine keyboard!') + elif not cli.config.flash.keymap: + cli.log.error('Could not determine keymap!') + + # Compile the firmware, if we're able to + if command: + cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) + if not cli.args.dry_run: + cli.echo('\n') + compile = cli.run(command, capture_output=False, stdin=DEVNULL) + return compile.returncode + + else: + cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') + return False diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 7da9df1d8a..10da5e7e8e 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -64,6 +64,54 @@ LEGACY_KEYCODES = { # Comment here is to force multiline formatting 'RESET': 'QK_BOOT' } +# Map VID:PID values to bootloaders +BOOTLOADER_VIDS_PIDS = { + 'atmel-dfu': { + ("03eb", "2fef"), # ATmega16U2 + ("03eb", "2ff0"), # ATmega32U2 + ("03eb", "2ff3"), # ATmega16U4 + ("03eb", "2ff4"), # ATmega32U4 + ("03eb", "2ff9"), # AT90USB64 + ("03eb", "2ffa"), # AT90USB162 + ("03eb", "2ffb") # AT90USB128 + }, + 'kiibohd': {("1c11", "b007")}, + 'stm32-dfu': { + ("1eaf", "0003"), # STM32duino + ("0483", "df11") # STM32 DFU + }, + 'apm32-dfu': {("314b", "0106")}, + 'gd32v-dfu': {("28e9", "0189")}, + 'bootloadhid': {("16c0", "05df")}, + 'usbasploader': {("16c0", "05dc")}, + 'usbtinyisp': {("1782", "0c9f")}, + 'md-boot': {("03eb", "6124")}, + 'caterina': { + # pid.codes shared PID + ("1209", "9203"), # Keyboardio Atreus 2 Bootloader + # Spark Fun Electronics + ("1b4f", "9203"), # Pro Micro 3V3/8MHz + ("1b4f", "9205"), # Pro Micro 5V/16MHz + ("1b4f", "9207"), # LilyPad 3V3/8MHz (and some Pro Micro clones) + # Pololu Electronics + ("1ffb", "0101"), # A-Star 32U4 + # Arduino SA + ("2341", "0036"), # Leonardo + ("2341", "0037"), # Micro + # Adafruit Industries LLC + ("239a", "000c"), # Feather 32U4 + ("239a", "000d"), # ItsyBitsy 32U4 3V3/8MHz + ("239a", "000e"), # ItsyBitsy 32U4 5V/16MHz + # dog hunter AG + ("2a03", "0036"), # Leonardo + ("2a03", "0037") # Micro + }, + 'hid-bootloader': { + ("03eb", "2067"), # QMK HID + ("16c0", "0478") # PJRC halfkay + } +} + # Common format strings DATE_FORMAT = '%Y-%m-%d' DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py new file mode 100644 index 0000000000..a9cf726b44 --- /dev/null +++ b/lib/python/qmk/flashers.py @@ -0,0 +1,203 @@ +import shutil +import time +import os +import signal + +import usb.core + +from qmk.constants import BOOTLOADER_VIDS_PIDS +from milc import cli + +# yapf: disable +_PID_TO_MCU = { + '2fef': 'atmega16u2', + '2ff0': 'atmega32u2', + '2ff3': 'atmega16u4', + '2ff4': 'atmega32u4', + '2ff9': 'at90usb64', + '2ffa': 'at90usb162', + '2ffb': 'at90usb128' +} + +AVRDUDE_MCU = { + 'atmega32a': 'm32', + 'atmega328p': 'm328p', + 'atmega328': 'm328', +} +# yapf: enable + + +class DelayedKeyboardInterrupt: + # Custom interrupt handler to delay the processing of Ctrl-C + # https://stackoverflow.com/a/21919644 + def __enter__(self): + self.signal_received = False + self.old_handler = signal.signal(signal.SIGINT, self.handler) + + def handler(self, sig, frame): + self.signal_received = (sig, frame) + + def __exit__(self, type, value, traceback): + signal.signal(signal.SIGINT, self.old_handler) + if self.signal_received: + self.old_handler(*self.signal_received) + + +# TODO: Make this more generic, so cli/doctor/check.py and flashers.py can share the code +def _check_dfu_programmer_version(): + # Return True if version is higher than 0.7.0: supports '--force' + check = cli.run(['dfu-programmer', '--version'], combined_output=True, timeout=5) + first_line = check.stdout.split('\n')[0] + version_number = first_line.split()[1] + maj, min_, bug = version_number.split('.') + if int(maj) >= 0 and int(min_) >= 7: + return True + else: + return False + + +def _find_bootloader(): + # To avoid running forever in the background, only look for bootloaders for 10min + start_time = time.time() + while time.time() - start_time < 600: + for bl in BOOTLOADER_VIDS_PIDS: + for vid, pid in BOOTLOADER_VIDS_PIDS[bl]: + vid_hex = int(f'0x{vid}', 0) + pid_hex = int(f'0x{pid}', 0) + with DelayedKeyboardInterrupt(): + # PyUSB does not like to be interrupted by Ctrl-C + # therefore we catch the interrupt with a custom handler + # and only process it once pyusb finished + dev = usb.core.find(idVendor=vid_hex, idProduct=pid_hex) + if dev: + if bl == 'atmel-dfu': + details = _PID_TO_MCU[pid] + elif bl == 'caterina': + details = (vid_hex, pid_hex) + elif bl == 'hid-bootloader': + if vid == '16c0' and pid == '0478': + details = 'halfkay' + else: + details = 'qmk-hid' + elif bl == 'stm32-dfu' or bl == 'apm32-dfu' or bl == 'gd32v-dfu' or bl == 'kiibohd': + details = (vid, pid) + else: + details = None + return (bl, details) + time.sleep(0.1) + return (None, None) + + +def _find_serial_port(vid, pid): + if 'windows' in cli.platform.lower(): + from serial.tools.list_ports_windows import comports + platform = 'windows' + else: + from serial.tools.list_ports_posix import comports + platform = 'posix' + + start_time = time.time() + # Caterina times out after 8 seconds + while time.time() - start_time < 8: + for port in comports(): + port, desc, hwid = port + if f'{vid:04x}:{pid:04x}' in hwid.casefold(): + if platform == 'windows': + time.sleep(1) + return port + else: + start_time = time.time() + # Wait until the port becomes writable before returning + while time.time() - start_time < 8: + if os.access(port, os.W_OK): + return port + else: + time.sleep(0.5) + return None + return None + + +def _flash_caterina(details, file): + port = _find_serial_port(details[0], details[1]) + if port: + cli.run(['avrdude', '-p', 'atmega32u4', '-c', 'avr109', '-U', f'flash:w:{file}:i', '-P', port], capture_output=False) + return False + else: + return True + + +def _flash_atmel_dfu(mcu, file): + force = '--force' if _check_dfu_programmer_version() else '' + cli.run(['dfu-programmer', mcu, 'erase', force], capture_output=False) + cli.run(['dfu-programmer', mcu, 'flash', force, file], capture_output=False) + cli.run(['dfu-programmer', mcu, 'reset'], capture_output=False) + + +def _flash_hid_bootloader(mcu, details, file): + if details == 'halfkay': + if shutil.which('teensy-loader-cli'): + cmd = 'teensy-loader-cli' + elif shutil.which('teensy_loader_cli'): + cmd = 'teensy_loader_cli' + + # Use 'hid_bootloader_cli' for QMK HID and as a fallback for HalfKay + if not cmd: + if shutil.which('hid_bootloader_cli'): + cmd = 'hid_bootloader_cli' + else: + return True + + cli.run([cmd, f'-mmcu={mcu}', '-w', '-v', file], capture_output=False) + + +def _flash_dfu_util(details, file): + # STM32duino + if details[0] == '1eaf' and details[1] == '0003': + cli.run(['dfu-util', '-a', '2', '-d', f'{details[0]}:{details[1]}', '-R', '-D', file], capture_output=False) + # kiibohd + elif details[0] == '1c11' and details[1] == 'b007': + cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-D', file], capture_output=False) + # STM32, APM32, or GD32V DFU + else: + cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-s', '0x08000000:leave', '-D', file], capture_output=False) + + +def _flash_isp(mcu, programmer, file): + programmer = 'usbasp' if programmer == 'usbasploader' else 'usbtiny' + # Check if the provide mcu has an avrdude-specific name, otherwise pass on what the user provided + mcu = AVRDUDE_MCU.get(mcu, mcu) + cli.run(['avrdude', '-p', mcu, '-c', programmer, '-U', f'flash:w:{file}:i'], capture_output=False) + + +def _flash_mdloader(file): + cli.run(['mdloader', '--first', '--download', file, '--restart'], capture_output=False) + + +def flasher(mcu, file): + bl, details = _find_bootloader() + # Add a small sleep to avoid race conditions + time.sleep(1) + if bl == 'atmel-dfu': + _flash_atmel_dfu(details, file.name) + elif bl == 'caterina': + if _flash_caterina(details, file.name): + return (True, "The Caterina bootloader was found but is not writable. Check 'qmk doctor' output for advice.") + elif bl == 'hid-bootloader': + if mcu: + if _flash_hid_bootloader(mcu, details, file.name): + return (True, "Please make sure 'teensy_loader_cli' or 'hid_bootloader_cli' is available on your system.") + else: + return (True, "Specifying the MCU with '-m' is necessary for HalfKay/HID bootloaders!") + elif bl == 'stm32-dfu' or bl == 'apm32-dfu' or bl == 'gd32v-dfu' or bl == 'kiibohd': + _flash_dfu_util(details, file.name) + elif bl == 'usbasploader' or bl == 'usbtinyisp': + if mcu: + _flash_isp(mcu, bl, file.name) + else: + return (True, "Specifying the MCU with '-m' is necessary for ISP flashing!") + elif bl == 'md-boot': + _flash_mdloader(file.name) + else: + return (True, "Known bootloader found but flashing not currently supported!") + + return (False, None) |