diff --git a/tools/licensing/apachize.py b/tools/licensing/apachize.py index 7c26e6350a..a86a6ce742 100755 --- a/tools/licensing/apachize.py +++ b/tools/licensing/apachize.py @@ -25,22 +25,30 @@ apache = r""" * ****************************************************************************/""" + def apachize(path, header): - relpath = os.path.relpath(path, os.environ['TOPDIR']) - header = re.sub('PATH', relpath, header) + relpath = os.path.relpath(path, os.environ["TOPDIR"]) + header = re.sub("PATH", relpath, header) with open(path) as f: s = f.read() - s = re.sub('(?i)/\*\*\*.+?(?:Copyright).+?\*\*\*+/', header, s, 1, re.DOTALL) + s = re.sub("(?i)/\*\*\*.+?(?:Copyright).+?\*\*\*+/", header, s, 1, re.DOTALL) print(s) -if (len(sys.argv) != 2): - print("Usage: ./apachize.py ", file = sys.stderr) - print("This will replace the license header of the passed file to that of Apache 2.0 and print it to stdout", file = sys.sterr) + +if len(sys.argv) != 2: + print("Usage: ./apachize.py ", file=sys.stderr) + print( + "This will replace the license header of the passed file to that of Apache 2.0 and print it to stdout", + file=sys.sterr, + ) sys.exit(2) -if (not 'TOPDIR' in os.environ): - print("Please define the TOPDIR environment variable to the full path to nuttx/ root", file = sys.stderr) +if not "TOPDIR" in os.environ: + print( + "Please define the TOPDIR environment variable to the full path to nuttx/ root", + file=sys.stderr, + ) sys.exit(2) apachize(sys.argv[1], apache) diff --git a/tools/licensing/check.py b/tools/licensing/check.py index 884d6faa47..837c48b4ad 100755 --- a/tools/licensing/check.py +++ b/tools/licensing/check.py @@ -34,146 +34,167 @@ author_mappings_json = None verbose_level = 0 color = True -def colored(s,c): + +def colored(s, c): if color: - return termcolor.colored(s,c) + return termcolor.colored(s, c) else: return s + def commit_attributions(c): - regex = re.compile('(?i)(?:by|from|author|Co-authored-by):? +(.+)') - return re.findall(regex, c['message']) + re.findall(regex,c['body']) + regex = re.compile("(?i)(?:by|from|author|Co-authored-by):? +(.+)") + return re.findall(regex, c["message"]) + re.findall(regex, c["body"]) + def get_headers(s): - return re.findall('(?i)/\*\*\*.+?(?:Copyright).+?\*\*\*+/', s, re.DOTALL) + return re.findall("(?i)/\*\*\*.+?(?:Copyright).+?\*\*\*+/", s, re.DOTALL) + def get_file(blob): try: - return subprocess.check_output(['git','cat-file','-p',blob], stderr=subprocess.DEVNULL).decode() + return subprocess.check_output( + ["git", "cat-file", "-p", blob], stderr=subprocess.DEVNULL + ).decode() except subprocess.CalledProcessError as err: return None + def header_authors(header): - results = re.findall('[Aa]uthors?: +(.+?) *(?:Redistribution)', header, re.DOTALL) - results = [re.split('\n[ *]+',result) for result in results] - results = sum(results,[]) # flatten - results = [re.sub('[Cc]opyright:?( ?.[Cc].)? *([12][0-9]{3}[,-]? ?)','',result) for result in results] - results = list(filter(lambda s: s != '', results)) # remove empty strings + results = re.findall("[Aa]uthors?: +(.+?) *(?:Redistribution)", header, re.DOTALL) + results = [re.split("\n[ *]+", result) for result in results] + results = sum(results, []) # flatten + results = [ + re.sub("[Cc]opyright:?( ?.[Cc].)? *([12][0-9]{3}[,-]? ?)", "", result) + for result in results + ] + results = list(filter(lambda s: s != "", results)) # remove empty strings return results + # Search for an author name in Apache's committers/non-committers # database. It will return (apacheID,name) if there's a match or # None if not. apacheID might be None if there's no Apache ID # for author -def search_for_cla(name): - for k,v in committers_json['committers'].items(): - if (v == name): - return (k,v) - if name in non_committers_json['non_committers']: - return (None,name) +def search_for_cla(name): + for k, v in committers_json["committers"].items(): + if v == name: + return (k, v) + + if name in non_committers_json["non_committers"]: + return (None, name) return None + # Returns the same as above, but this takes an author # (which may include an email include an email used # to look for alternative author names for this person) + def author_has_cla(author): - if ('@' in author): - matches = re.match('^(.+?)(?: +([^ ]+@[^ ]+ *))$', author) - if (not matches): - return None # found an '@' but it wasn't an email, so this is most likely not really an author + if "@" in author: + matches = re.match("^(.+?)(?: +([^ ]+@[^ ]+ *))$", author) + if not matches: + return None # found an '@' but it wasn't an email, so this is most likely not really an author name = matches.group(1) - email = matches.group(2).lstrip('<').rstrip('>') + email = matches.group(2).lstrip("<").rstrip(">") else: name = author.strip() email = None - vvvprint('name: %s email: %s' % (name,email if email else '?')) + vvvprint("name: %s email: %s" % (name, email if email else "?")) # first look for name directly result = search_for_cla(name) - if (result): + if result: return result - + # otherwise, get all available alternative names for author # and look for each - if (email and (email in author_mappings_json)): + if email and (email in author_mappings_json): result = search_for_cla(author_mappings_json[email]) - if (result): + if result: return result - + # Nothing matched return None + def header_copyrights(header): - results = re.findall(' \* *[Cc]opyright:?(?: ?.[Cc].)? *(?:[12][0-9]{3}[,-]? ?)* *(.+)', header) - return [re.sub('(. )?[Aa]ll rights reserved.?','',result) for result in results] + results = re.findall( + " \* *[Cc]opyright:?(?: ?.[Cc].)? *(?:[12][0-9]{3}[,-]? ?)* *(.+)", header + ) + return [re.sub("(. )?[Aa]ll rights reserved.?", "", result) for result in results] + def report_cla(author): cla = author_has_cla(author) if cla: - (apacheid,name) = cla - print(colored('✓','green'), end = ' ') + (apacheid, name) = cla + print(colored("✓", "green"), end=" ") else: apacheid = None - print(colored('✗','red'), end = ' ') + print(colored("✗", "red"), end=" ") if apacheid: - print('%s (ID: %s)' % (author, apacheid)) + print("%s (ID: %s)" % (author, apacheid)) else: print(author) + def analyze(j): complete_attributions = set() complete_authors = set() complete_copyrights = set() - vprint('file has %i commits' % len(j)) + vprint("file has %i commits" % len(j)) for commit in j: authors = set() - - vprint(colored('-','yellow')) - vprint(colored('commit: ', 'green') + commit['commit']) - vprint(colored('blob: ', 'green') + commit['blob']) - vprint(colored('date: ','green') + commit['date']) - vprint(colored('author: ','green') + ('%s <%s>' % (commit['author'], commit['author-email']))) + + vprint(colored("-", "yellow")) + vprint(colored("commit: ", "green") + commit["commit"]) + vprint(colored("blob: ", "green") + commit["blob"]) + vprint(colored("date: ", "green") + commit["date"]) + vprint( + colored("author: ", "green") + + ("%s <%s>" % (commit["author"], commit["author-email"])) + ) attributions = commit_attributions(commit) - if (len(attributions) > 0): - vprint(colored('attributions:','green')) + if len(attributions) > 0: + vprint(colored("attributions:", "green")) for attribution in attributions: vprint(attribution) complete_attributions |= set(attributions) - complete_authors |= set([commit['author'] + ' ' + commit['author-email']]) + complete_authors |= set([commit["author"] + " " + commit["author-email"]]) - # skip deletion commits + # skip deletion commits - vprint(colored('blob:','green'), end = ' ') - if (commit['blob'] == '0000000000000000000000000000000000000000'): - vprint('zero (deletion)') - continue + vprint(colored("blob:", "green"), end=" ") + if commit["blob"] == "0000000000000000000000000000000000000000": + vprint("zero (deletion)") + continue - file_contents = get_file(commit['blob']) + file_contents = get_file(commit["blob"]) # skip inaccessible blobs (probably lived in a submodule) - - if (not file_contents): - vprint('inaccessible') + if not file_contents: + vprint("inaccessible") continue else: - vprint('available') + vprint("available") headers = get_headers(file_contents) - vprint(colored('header authors:','green')) + vprint(colored("header authors:", "green")) for header in headers: ha = header_authors(header) authors |= set(ha) @@ -181,77 +202,92 @@ def analyze(j): complete_authors |= set(authors) - vprint(colored('header copyrights:','green')) + vprint(colored("header copyrights:", "green")) copyrights = set() for header in headers: hc = header_copyrights(header) copyrights |= set(hc) vprint(hc) - - vprint(colored('commit description:','green')) - vprint(commit['message']) - if commit['body']: - vprint(colored('commit msg body:','green')) - vprint(commit['body']) + vprint(colored("commit description:", "green")) + vprint(commit["message"]) - vvprint(colored('headers:','green')) + if commit["body"]: + vprint(colored("commit msg body:", "green")) + vprint(commit["body"]) + + vvprint(colored("headers:", "green")) for header in headers: vvprint(header) complete_copyrights |= copyrights - vprint(colored("----\n",'yellow')) - - print(colored("COMPLETE REPORT:",'blue')) - print(colored("attributions:",'green')) - if (len(complete_attributions) == 0): + vprint(colored("----\n", "yellow")) + + print(colored("COMPLETE REPORT:", "blue")) + print(colored("attributions:", "green")) + if len(complete_attributions) == 0: print("*none detected*") else: for attribution in complete_attributions: report_cla(attribution) - print(colored("authors:",'green')) + print(colored("authors:", "green")) for author in complete_authors: report_cla(author) - print(colored("copyrights:",'green')) - print('\n'.join(complete_copyrights)) + print(colored("copyrights:", "green")) + print("\n".join(complete_copyrights)) + def print_help(): print("Usage: check.py [-v] [-n] \n") - print(" -v\tIncrease verbosity (add up to three times)\n" - " -n\tDo not use color for output") + print( + " -v\tIncrease verbosity (add up to three times)\n" + " -n\tDo not use color for output" + ) + def vprint(*args, **kwargs): - if (verbose_level > 0): + if verbose_level > 0: print(*args, **kwargs) + def vvprint(*args, **kwargs): - if (verbose_level > 1): + if verbose_level > 1: print(*args, **kwargs) + def vvvprint(*args, **kwargs): - if (verbose_level > 2): + if verbose_level > 2: print(*args, **kwargs) + ##### # First try to load the CLAs JSONs: try: - with open(os.path.dirname(os.path.abspath(__file__)) + '/icla-info.json','r') as file: + with open( + os.path.dirname(os.path.abspath(__file__)) + "/icla-info.json", "r" + ) as file: committers_json = json.load(file) - with open(os.path.dirname(os.path.abspath(__file__)) + '/icla-info_noid.json','r') as file: + with open( + os.path.dirname(os.path.abspath(__file__)) + "/icla-info_noid.json", "r" + ) as file: non_committers_json = json.load(file) except: - print('Could not open CLA JSON files, please read README.md for download instructions') + print( + "Could not open CLA JSON files, please read README.md for download instructions" + ) sys.exit(2) # Open author mappings JSON -with open(os.path.dirname(os.path.abspath(__file__)) + '/author_mappings.json','r') as file: +with open( + os.path.dirname(os.path.abspath(__file__)) + "/author_mappings.json", "r" +) as file: author_mappings_json = json.load(file) try: @@ -260,7 +296,7 @@ except getopt.GetoptError: print_help() sys.exit(2) for opt, arg in opts: - if (opt == "-h"): + if opt == "-h": print_help() sys.exit() elif opt == "-v": @@ -268,7 +304,7 @@ for opt, arg in opts: elif opt == "-n": color = False -if (len(args) != 1): +if len(args) != 1: print_help() sys.exit(2) @@ -278,10 +314,10 @@ if not f: print_help() sys.exit(2) -if (f == '-'): +if f == "-": j = json.load(sys.stdin) else: - with open(f, 'r') as file: - j = json.load(file) + with open(f, "r") as file: + j = json.load(file) analyze(j)