From 6e83f08f77c621dd77339021301efe73c3d05dcc Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Sun, 29 Dec 2024 22:13:45 -0800 Subject: [PATCH] rework --- lib/parallel_tests/cli.rb | 3 ++ lib/parallel_tests/test/runner.rb | 73 ++++++++++++------------------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 54367544..ceb07744 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -290,6 +290,7 @@ def parse_options!(argv) opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } opts.on("--fail-fast", "Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported") { options[:fail_fast] = true } + opts.on("--cmd-length-limit", Integer, "Limit commands to this length by batching files that are being tested") { |limit| options[:cmd_length_limit] = limit } opts.on("--verbose", "Print debug output") { options[:verbose] = true } opts.on("--verbose-command", "Combines options --verbose-process-command and --verbose-rerun-command") { options.merge! verbose_process_command: true, verbose_rerun_command: true } opts.on("--verbose-process-command", "Print the command that will be executed by each process before it begins") { options[:verbose_process_command] = true } @@ -329,6 +330,8 @@ def parse_options!(argv) options[:group_by] ||= :filesize if options[:only_group] + options[:cmd_length_limit] ||= 8192 if WINDOWS + if options[:group_by] == :found && options[:single_process] raise "--group-by found and --single-process are not supported" end diff --git a/lib/parallel_tests/test/runner.rb b/lib/parallel_tests/test/runner.rb index 9b94eaed..e00236ab 100644 --- a/lib/parallel_tests/test/runner.rb +++ b/lib/parallel_tests/test/runner.rb @@ -85,41 +85,34 @@ def tests_with_size(tests, options) tests end - def process_in_batches(cmd, os_cmd_length_limit, tests) - # Filter elements not starting with value in tests to retain in each batch - split_elements = cmd.partition { |s| s.start_with?(tests) } - - # elements that needs to be checked for length and sliced into batches - non_retained_elements = split_elements.first - - # common parameters for each batch - retained_elements = split_elements.last - + def split_command_batches(cmd, limit, first_test) batches = [] - index = 0 - while index < non_retained_elements.length - batch = retained_elements.dup - total_length = batch.map(&:length).sum - total_length += batch.size # account for spaces between elements - - while index < non_retained_elements.length - current_element_length = non_retained_elements[index].length - current_element_length += 1 # account for spaces between elements - - # Check if the current element can be added without exceeding the os cmd length limit - break unless total_length + current_element_length <= os_cmd_length_limit - - batch << non_retained_elements[index] - total_length += current_element_length - index += 1 + start = cmd.index(first_test) + base = cmd[0...start] + raise "base is too long" if base.shelljoin.size > limit + batch = base.dup # first batch + cmd[start..].each do |arg| + if (batch + [arg]).shelljoin.size > limit + batches << batch + batch = base.dup end - - batches << batch + batch << arg end - + batches << batch # last batch batches end + def combine_batched_results(results) + result = results[0] + results[1..].each do |res| + result[:stdout] = result[:stdout].to_s + res[:stdout].to_s + result[:exit_status] = [res[:exit_status], result[:exit_status]].max + # adding all files back in, not using original cmd to show what was actually run + result[:command] = result[:command] | res[:command] + end + result + end + def execute_command(cmd, process_number, num_processes, options) number = test_env_number(process_number, options).to_s env = (options[:env] || {}).merge( @@ -134,24 +127,14 @@ def execute_command(cmd, process_number, num_processes, options) print_command(cmd, env) if report_process_command?(options) && !options[:serialize_stdout] - result = [] - process_in_batches(cmd, 8191, options[:files].first).map do |subcmd| - result << execute_command_and_capture_output(env, subcmd, options) - end - - # combine the output of result array into a single Hash - combined_result = {} - result.each do |res| - if combined_result.empty? - combined_result = res - else - combined_result[:stdout] = combined_result[:stdout].to_s + res[:stdout].to_s - combined_result[:exit_status] = res[:exit_status] > combined_result[:exit_status] ? res[:exit_status] : combined_result[:exit_status] # keep the max - combined_result[:command] = combined_result[:command] | res[:command] + if (limit = options[:cmd_length_limit]) && limit != 0 + results = split_command_batches(cmd, limit, options[:files].first).map do |subcmd| + execute_command_and_capture_output(env, subcmd, options) end + combine_batched_results(results) + else + execute_command_and_capture_output(env, cmd, options) end - - combined_result end def print_command(command, env)