From: Simon Glass <sjg@chromium.org> This script has conflict markers in it so comes up as an unwanted match when searching them. Remove the file since it is not used in U-Boot. Signed-off-by: Simon Glass <sjg@chromium.org> --- .../mbedtls/tests/scripts/check_files.py | 537 ------------------ 1 file changed, 537 deletions(-) delete mode 100755 lib/mbedtls/external/mbedtls/tests/scripts/check_files.py diff --git a/lib/mbedtls/external/mbedtls/tests/scripts/check_files.py b/lib/mbedtls/external/mbedtls/tests/scripts/check_files.py deleted file mode 100755 index d5a4b921e4f..00000000000 --- a/lib/mbedtls/external/mbedtls/tests/scripts/check_files.py +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - -""" -This script checks the current state of the source code for minor issues, -including incorrect file permissions, presence of tabs, non-Unix line endings, -trailing whitespace, and presence of UTF-8 BOM. -Note: requires python 3, must be run from Mbed TLS root. -""" - -import argparse -import codecs -import inspect -import logging -import os -import re -import subprocess -import sys -try: - from typing import FrozenSet, Optional, Pattern # pylint: disable=unused-import -except ImportError: - pass - -import scripts_path # pylint: disable=unused-import -from mbedtls_dev import build_tree - - -class FileIssueTracker: - """Base class for file-wide issue tracking. - - To implement a checker that processes a file as a whole, inherit from - this class and implement `check_file_for_issue` and define ``heading``. - - ``suffix_exemptions``: files whose name ends with a string in this set - will not be checked. - - ``path_exemptions``: files whose path (relative to the root of the source - tree) matches this regular expression will not be checked. This can be - ``None`` to match no path. Paths are normalized and converted to ``/`` - separators before matching. - - ``heading``: human-readable description of the issue - """ - - suffix_exemptions = frozenset() #type: FrozenSet[str] - path_exemptions = None #type: Optional[Pattern[str]] - # heading must be defined in derived classes. - # pylint: disable=no-member - - def __init__(self): - self.files_with_issues = {} - - @staticmethod - def normalize_path(filepath): - """Normalize ``filepath`` with / as the directory separator.""" - filepath = os.path.normpath(filepath) - # On Windows, we may have backslashes to separate directories. - # We need slashes to match exemption lists. - seps = os.path.sep - if os.path.altsep is not None: - seps += os.path.altsep - return '/'.join(filepath.split(seps)) - - def should_check_file(self, filepath): - """Whether the given file name should be checked. - - Files whose name ends with a string listed in ``self.suffix_exemptions`` - or whose path matches ``self.path_exemptions`` will not be checked. - """ - for files_exemption in self.suffix_exemptions: - if filepath.endswith(files_exemption): - return False - if self.path_exemptions and \ - re.match(self.path_exemptions, self.normalize_path(filepath)): - return False - return True - - def check_file_for_issue(self, filepath): - """Check the specified file for the issue that this class is for. - - Subclasses must implement this method. - """ - raise NotImplementedError - - def record_issue(self, filepath, line_number): - """Record that an issue was found at the specified location.""" - if filepath not in self.files_with_issues.keys(): - self.files_with_issues[filepath] = [] - self.files_with_issues[filepath].append(line_number) - - def output_file_issues(self, logger): - """Log all the locations where the issue was found.""" - if self.files_with_issues.values(): - logger.info(self.heading) - for filename, lines in sorted(self.files_with_issues.items()): - if lines: - logger.info("{}: {}".format( - filename, ", ".join(str(x) for x in lines) - )) - else: - logger.info(filename) - logger.info("") - -BINARY_FILE_PATH_RE_LIST = [ - r'docs/.*\.pdf\Z', - r'docs/.*\.png\Z', - r'programs/fuzz/corpuses/[^.]+\Z', - r'tests/data_files/[^.]+\Z', - r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z', - r'tests/data_files/.*\.req\.[^/]+\Z', - r'tests/data_files/.*malformed[^/]+\Z', - r'tests/data_files/format_pkcs12\.fmt\Z', - r'tests/data_files/.*\.bin\Z', -] -BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST)) - -class LineIssueTracker(FileIssueTracker): - """Base class for line-by-line issue tracking. - - To implement a checker that processes files line by line, inherit from - this class and implement `line_with_issue`. - """ - - # Exclude binary files. - path_exemptions = BINARY_FILE_PATH_RE - - def issue_with_line(self, line, filepath, line_number): - """Check the specified line for the issue that this class is for. - - Subclasses must implement this method. - """ - raise NotImplementedError - - def check_file_line(self, filepath, line, line_number): - if self.issue_with_line(line, filepath, line_number): - self.record_issue(filepath, line_number) - - def check_file_for_issue(self, filepath): - """Check the lines of the specified file. - - Subclasses must implement the ``issue_with_line`` method. - """ - with open(filepath, "rb") as f: - for i, line in enumerate(iter(f.readline, b"")): - self.check_file_line(filepath, line, i + 1) - - -def is_windows_file(filepath): - _root, ext = os.path.splitext(filepath) - return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj') - - -class ShebangIssueTracker(FileIssueTracker): - """Track files with a bad, missing or extraneous shebang line. - - Executable scripts must start with a valid shebang (#!) line. - """ - - heading = "Invalid shebang line:" - - # Allow either /bin/sh, /bin/bash, or /usr/bin/env. - # Allow at most one argument (this is a Linux limitation). - # For sh and bash, the argument if present must be options. - # For env, the argument must be the base name of the interpreter. - _shebang_re = re.compile(rb'^#! ?(?:/bin/(bash|sh)(?: -[^\n ]*)?' - rb'|/usr/bin/env ([^\n /]+))$') - _extensions = { - b'bash': 'sh', - b'perl': 'pl', - b'python3': 'py', - b'sh': 'sh', - } - - path_exemptions = re.compile(r'tests/scripts/quiet/.*') - - def is_valid_shebang(self, first_line, filepath): - m = re.match(self._shebang_re, first_line) - if not m: - return False - interpreter = m.group(1) or m.group(2) - if interpreter not in self._extensions: - return False - if not filepath.endswith('.' + self._extensions[interpreter]): - return False - return True - - def check_file_for_issue(self, filepath): - is_executable = os.access(filepath, os.X_OK) - with open(filepath, "rb") as f: - first_line = f.readline() - if first_line.startswith(b'#!'): - if not is_executable: - # Shebang on a non-executable file - self.files_with_issues[filepath] = None - elif not self.is_valid_shebang(first_line, filepath): - self.files_with_issues[filepath] = [1] - elif is_executable: - # Executable without a shebang - self.files_with_issues[filepath] = None - - -class EndOfFileNewlineIssueTracker(FileIssueTracker): - """Track files that end with an incomplete line - (no newline character at the end of the last line).""" - - heading = "Missing newline at end of file:" - - path_exemptions = BINARY_FILE_PATH_RE - - def check_file_for_issue(self, filepath): - with open(filepath, "rb") as f: - try: - f.seek(-1, 2) - except OSError: - # This script only works on regular files. If we can't seek - # 1 before the end, it means that this position is before - # the beginning of the file, i.e. that the file is empty. - return - if f.read(1) != b"\n": - self.files_with_issues[filepath] = None - - -class Utf8BomIssueTracker(FileIssueTracker): - """Track files that start with a UTF-8 BOM. - Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM.""" - - heading = "UTF-8 BOM present:" - - suffix_exemptions = frozenset([".vcxproj", ".sln"]) - path_exemptions = BINARY_FILE_PATH_RE - - def check_file_for_issue(self, filepath): - with open(filepath, "rb") as f: - if f.read().startswith(codecs.BOM_UTF8): - self.files_with_issues[filepath] = None - - -class UnicodeIssueTracker(LineIssueTracker): - """Track lines with invalid characters or invalid text encoding.""" - - heading = "Invalid UTF-8 or forbidden character:" - - # Only allow valid UTF-8, and only other explicitly allowed characters. - # We deliberately exclude all characters that aren't a simple non-blank, - # non-zero-width glyph, apart from a very small set (tab, ordinary space, - # line breaks, "basic" no-break space and soft hyphen). In particular, - # non-ASCII control characters, combinig characters, and Unicode state - # changes (e.g. right-to-left text) are forbidden. - # Note that we do allow some characters with a risk of visual confusion, - # for example '-' (U+002D HYPHEN-MINUS) vs '' (U+00AD SOFT HYPHEN) vs - # '‐' (U+2010 HYPHEN), or 'A' (U+0041 LATIN CAPITAL LETTER A) vs - # 'Α' (U+0391 GREEK CAPITAL LETTER ALPHA). - GOOD_CHARACTERS = ''.join([ - '\t\n\r -~', # ASCII (tabs and line endings are checked separately) - '\u00A0-\u00FF', # Latin-1 Supplement (for NO-BREAK SPACE and punctuation) - '\u2010-\u2027\u2030-\u205E', # General Punctuation (printable) - '\u2070\u2071\u2074-\u208E\u2090-\u209C', # Superscripts and Subscripts - '\u2190-\u21FF', # Arrows - '\u2200-\u22FF', # Mathematical Symbols - '\u2500-\u257F' # Box Drawings characters used in markdown trees - ]) - # Allow any of the characters and ranges above, and anything classified - # as a word constituent. - GOOD_CHARACTERS_RE = re.compile(r'[\w{}]+\Z'.format(GOOD_CHARACTERS)) - - def issue_with_line(self, line, _filepath, line_number): - try: - text = line.decode('utf-8') - except UnicodeDecodeError: - return True - if line_number == 1 and text.startswith('\uFEFF'): - # Strip BOM (U+FEFF ZERO WIDTH NO-BREAK SPACE) at the beginning. - # Which files are allowed to have a BOM is handled in - # Utf8BomIssueTracker. - text = text[1:] - return not self.GOOD_CHARACTERS_RE.match(text) - -class UnixLineEndingIssueTracker(LineIssueTracker): - """Track files with non-Unix line endings (i.e. files with CR).""" - - heading = "Non-Unix line endings:" - - def should_check_file(self, filepath): - if not super().should_check_file(filepath): - return False - return not is_windows_file(filepath) - - def issue_with_line(self, line, _filepath, _line_number): - return b"\r" in line - - -class WindowsLineEndingIssueTracker(LineIssueTracker): - """Track files with non-Windows line endings (i.e. CR or LF not in CRLF).""" - - heading = "Non-Windows line endings:" - - def should_check_file(self, filepath): - if not super().should_check_file(filepath): - return False - return is_windows_file(filepath) - - def issue_with_line(self, line, _filepath, _line_number): - return not line.endswith(b"\r\n") or b"\r" in line[:-2] - - -class TrailingWhitespaceIssueTracker(LineIssueTracker): - """Track lines with trailing whitespace.""" - - heading = "Trailing whitespace:" - suffix_exemptions = frozenset([".dsp", ".md"]) - - def issue_with_line(self, line, _filepath, _line_number): - return line.rstrip(b"\r\n") != line.rstrip() - - -class TabIssueTracker(LineIssueTracker): - """Track lines with tabs.""" - - heading = "Tabs present:" - suffix_exemptions = frozenset([ - ".make", - ".pem", # some openssl dumps have tabs - ".sln", - "/.gitmodules", - "/Makefile", - "/Makefile.inc", - "/generate_visualc_files.pl", - ]) - - def issue_with_line(self, line, _filepath, _line_number): - return b"\t" in line - - -class MergeArtifactIssueTracker(LineIssueTracker): - """Track lines with merge artifacts. - These are leftovers from a ``git merge`` that wasn't fully edited.""" - - heading = "Merge artifact:" - - def issue_with_line(self, line, _filepath, _line_number): - # Detect leftover git conflict markers. - if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '): - return True - if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3 - return True - if line.rstrip(b'\r\n') == b'=======' and \ - not _filepath.endswith('.md'): - return True - return False - - -def this_location(): - frame = inspect.currentframe() - assert frame is not None - info = inspect.getframeinfo(frame) - return os.path.basename(info.filename), info.lineno -THIS_FILE_BASE_NAME, LINE_NUMBER_BEFORE_LICENSE_ISSUE_TRACKER = this_location() - -class LicenseIssueTracker(LineIssueTracker): - """Check copyright statements and license indications. - - This class only checks that statements are correct if present. It does - not enforce the presence of statements in each file. - """ - - heading = "License issue:" - - LICENSE_EXEMPTION_RE_LIST = [ - # Third-party code, other than whitelisted third-party modules, - # may be under a different license. - r'3rdparty/(?!(p256-m)/.*)', - # Documentation explaining the license may have accidental - # false positives. - r'(ChangeLog|LICENSE|[-0-9A-Z_a-z]+\.md)\Z', - # Files imported from TF-M, and not used except in test builds, - # may be under a different license. - r'configs/ext/crypto_config_profile_medium\.h\Z', - r'configs/ext/tfm_mbedcrypto_config_profile_medium\.h\Z', - r'configs/ext/README\.md\Z', - # Third-party file. - r'dco\.txt\Z', - ] - path_exemptions = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST + - LICENSE_EXEMPTION_RE_LIST)) - - COPYRIGHT_HOLDER = rb'The Mbed TLS Contributors' - # Catch "Copyright foo", "Copyright (C) foo", "Copyright © foo", etc. - COPYRIGHT_RE = re.compile(rb'.*\bcopyright\s+((?:\w|\s|[()]|[^ -~])*\w)', re.I) - - SPDX_HEADER_KEY = b'SPDX-License-Identifier' - LICENSE_IDENTIFIER = b'Apache-2.0 OR GPL-2.0-or-later' - SPDX_RE = re.compile(br'.*?(' + - re.escape(SPDX_HEADER_KEY) + - br')(:\s*(.*?)\W*\Z|.*)', re.I) - - LICENSE_MENTION_RE = re.compile(rb'.*(?:' + rb'|'.join([ - rb'Apache License', - rb'General Public License', - ]) + rb')', re.I) - - def __init__(self): - super().__init__() - # Record what problem was caused. We can't easily report it due to - # the structure of the script. To be fixed after - # https://github.com/Mbed-TLS/mbedtls/pull/2506 - self.problem = None - - def issue_with_line(self, line, filepath, line_number): - #pylint: disable=too-many-return-statements - - # Use endswith() rather than the more correct os.path.basename() - # because experimentally, it makes a significant difference to - # the running time. - if filepath.endswith(THIS_FILE_BASE_NAME) and \ - line_number > LINE_NUMBER_BEFORE_LICENSE_ISSUE_TRACKER: - # Avoid false positives from the code in this class. - # Also skip the rest of this file, which is highly unlikely to - # contain any problematic statements since we put those near the - # top of files. - return False - - m = self.COPYRIGHT_RE.match(line) - if m and m.group(1) != self.COPYRIGHT_HOLDER: - self.problem = 'Invalid copyright line' - return True - - m = self.SPDX_RE.match(line) - if m: - if m.group(1) != self.SPDX_HEADER_KEY: - self.problem = 'Misspelled ' + self.SPDX_HEADER_KEY.decode() - return True - if not m.group(3): - self.problem = 'Improperly formatted SPDX license identifier' - return True - if m.group(3) != self.LICENSE_IDENTIFIER: - self.problem = 'Wrong SPDX license identifier' - return True - - m = self.LICENSE_MENTION_RE.match(line) - if m: - self.problem = 'Suspicious license mention' - return True - - return False - - -class IntegrityChecker: - """Sanity-check files under the current directory.""" - - def __init__(self, log_file): - """Instantiate the sanity checker. - Check files under the current directory. - Write a report of issues to log_file.""" - build_tree.check_repo_path() - self.logger = None - self.setup_logger(log_file) - self.issues_to_check = [ - ShebangIssueTracker(), - EndOfFileNewlineIssueTracker(), - Utf8BomIssueTracker(), - UnicodeIssueTracker(), - UnixLineEndingIssueTracker(), - WindowsLineEndingIssueTracker(), - TrailingWhitespaceIssueTracker(), - TabIssueTracker(), - MergeArtifactIssueTracker(), - LicenseIssueTracker(), - ] - - def setup_logger(self, log_file, level=logging.INFO): - """Log to log_file if provided, or to stderr if None.""" - self.logger = logging.getLogger() - self.logger.setLevel(level) - if log_file: - handler = logging.FileHandler(log_file) - self.logger.addHandler(handler) - else: - console = logging.StreamHandler() - self.logger.addHandler(console) - - @staticmethod - def collect_files(): - """Return the list of files to check. - - These are the regular files commited into Git. - """ - bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) - bytes_filepaths = bytes_output.split(b'\0')[:-1] - ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) - # Filter out directories. Normally Git doesn't list directories - # (it only knows about the files inside them), but there is - # at least one case where 'git ls-files' includes a directory: - # submodules. Just skip submodules (and any other directories). - ascii_filepaths = [fp for fp in ascii_filepaths - if os.path.isfile(fp)] - # Prepend './' to files in the top-level directory so that - # something like `'/Makefile' in fp` matches in the top-level - # directory as well as in subdirectories. - return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp) - for fp in ascii_filepaths] - - def check_files(self): - """Check all files for all issues.""" - for issue_to_check in self.issues_to_check: - for filepath in self.collect_files(): - if issue_to_check.should_check_file(filepath): - issue_to_check.check_file_for_issue(filepath) - - def output_issues(self): - """Log the issues found and their locations. - - Return 1 if there were issues, 0 otherwise. - """ - integrity_return_code = 0 - for issue_to_check in self.issues_to_check: - if issue_to_check.files_with_issues: - integrity_return_code = 1 - issue_to_check.output_file_issues(self.logger) - return integrity_return_code - - -def run_main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "-l", "--log_file", type=str, help="path to optional output log", - ) - check_args = parser.parse_args() - integrity_check = IntegrityChecker(check_args.log_file) - integrity_check.check_files() - return_code = integrity_check.output_issues() - sys.exit(return_code) - - -if __name__ == "__main__": - run_main() -- 2.43.0