Re: [PATCH v2 3/4] patman: Run get_maintainer.pl in parallel

Hi,

On Sun, Feb 19, 2023 at 3:50 PM Simon Glass sjg@chromium.org wrote:

This script can take ages on some series. Try to limit the time by using threads. If a few stubborn patches remain, show progress so the user has some idea what is going on.

Signed-off-by: Simon Glass sjg@chromium.org

(no changes since v1)

tools/patmanu/func_test.py | 2 ++ tools/patmanu/series.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/tools/patmanu/func_test.py b/tools/patmanu/func_test.py index 238fd5b6100..48109ae5725 100644 --- a/tools/patmanu/func_test.py +++ b/tools/patmanu/func_test.py @@ -240,6 +240,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Change log missing for v3', next(lines)) self.assertEqual('Change log for unknown version v4', next(lines)) self.assertEqual("Alias 'pci' not found", next(lines))

•    while next(lines) != 'Cc processing complete':
  

•        pass
     self.assertIn('Dry run', next(lines))
     self.assertEqual('', next(lines))
     self.assertIn('Send a total of %d patches' % count, next(lines))
  

diff --git a/tools/patmanu/series.py b/tools/patmanu/series.py index 8ead87ef53e..e7a5f91da87 100644 --- a/tools/patmanu/series.py +++ b/tools/patmanu/series.py @@ -5,8 +5,11 @@ from __future__ import print_function

import collections +import concurrent.futures import itertools import os +import sys +import time

from patmanu import get_maintainer from patmanu import gitutil @@ -302,10 +305,34 @@ class Series(dict): fd = open(fname, 'w', encoding='utf-8') all_ccs = [] all_skips = set()

•    with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
  

•        for i, commit in enumerate(self.commits):
  

•            commit.seq = i
  

•            commit.future = executor.submit(
  

•                self.GetCcForCommit, commit, process_tags, warn_on_error,
  

•                add_maintainers, limit, get_maintainer_script, all_skips)
  

•        # Show progress any commits that are taking forever
  

•        lastlen = 0
  

•        while True:
  

•            left = [commit for commit in self.commits
  

•                    if not commit.future.done()]
  

•            if not left:
  

•                break
  

•            names = ', '.join(f'{c.seq + 1}:{c.subject}'
  

•                              for c in left[:2])
  

•            out = f'\r{len(left)} remaining: {names}'[:79]
  

•            spaces = ' ' * (lastlen - len(out))
  

•            if lastlen:  # Don't print anything the first time
  

•                print(out, spaces, end='')
  

•                sys.stdout.flush()
  

•            lastlen = len(out)
  

•            time.sleep(.25)
  

•        print(f'\rdone{" " * lastlen}\r', end='')
  

•        print('Cc processing complete')
  

•    for commit in self.commits:
  

•        cc = self.GetCcForCommit(commit, process_tags, warn_on_error,
  

•                                 add_maintainers, limit,
  

•                                 get_maintainer_script, all_skips)
  

•        cc = commit.future.result()
  

I've never used "concurrent.futures" before, but looks reasonable to me.

Reviewed-by: Douglas Anderson dianders@chromium.org