From 3b28178deb7a42a2df8c19b9bf190bb5da732b33 Mon Sep 17 00:00:00 2001 From: ruro Date: Wed, 18 Aug 2021 01:46:59 +0300 Subject: `--parallel` improvements (#13800) * improve make parallel jobs support * document the --parallel option * disable the output-sync for interactive targets --- lib/python/qmk/cli/compile.py | 2 +- lib/python/qmk/cli/flash.py | 2 +- lib/python/qmk/cli/multibuild.py | 6 +++--- lib/python/qmk/commands.py | 23 ++++++++++++++++++++--- 4 files changed, 25 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7a45e77214..acbd778649 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,7 @@ from qmk.keymap import keymap_completer @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 1b2932a5b2..c2d9e09c69 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -38,7 +38,7 @@ def print_bootloader_help(): @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('QMK Flash.') diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py index bdb0b493c8..85ed0fa1e9 100755 --- a/lib/python/qmk/cli/multibuild.py +++ b/lib/python/qmk/cli/multibuild.py @@ -10,7 +10,7 @@ from subprocess import DEVNULL from milc import cli from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make +from qmk.commands import _find_make, get_make_parallel_args import qmk.keyboard import qmk.keymap @@ -28,7 +28,7 @@ def _is_split(keyboard_name): return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") @@ -80,7 +80,7 @@ all: {keyboard_safe}_binary ) # yapf: enable - cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) + cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) # Check for failures failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 8c66228b2b..421453d837 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -52,7 +52,7 @@ def create_make_target(target, parallel=1, **env_vars): for key, value in env_vars.items(): env.append(f'{key}={value}') - return [make_cmd, '-j', str(parallel), *env, target] + return [make_cmd, *get_make_parallel_args(parallel), *env, target] def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): @@ -112,6 +112,24 @@ def get_git_version(current_time, repo_dir='.', check_dir='.'): return current_time +def get_make_parallel_args(parallel=1): + """Returns the arguments for running the specified number of parallel jobs. + """ + parallel_args = [] + + if int(parallel) <= 0: + # 0 or -1 means -j without argument (unlimited jobs) + parallel_args.append('--jobs') + else: + parallel_args.append('--jobs=' + str(parallel)) + + if int(parallel) != 1: + # If more than 1 job is used, synchronize parallel output by target + parallel_args.append('--output-sync=target') + + return parallel_args + + def create_version_h(skip_git=False, skip_all=False): """Generate version.h contents """ @@ -185,8 +203,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va make_command.append('-s') make_command.extend([ - '-j', - str(parallel), + *get_make_parallel_args(parallel), '-r', '-R', '-f', -- cgit v1.2.3