PRESUBMIT.py

# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# pragma pylint: disable=invalid-name, import-error
"""
This file acts as a git pre-submission checker, and uses the support tooling
from depot_tools to check a variety of style and programming requirements.
"""

from collections import namedtuple
import os
import re
import sys


_REPO_PATH = os.path.dirname(os.path.realpath('__file__'))
_IMPORT_SUBFOLDERS = [
    'tools',
    os.path.join('tools', 'licenses'),
    os.path.join('buildtools', 'checkdeps')
]

# git-cl upload is not compatible with __init__.py based subfolder imports, so
# we extend the system path instead.
#
# TODO(crbug.com/1502599): Migrate these modules to depot_tools and import from
# there.
sys.path.extend(os.path.join(_REPO_PATH, p) for p in _IMPORT_SUBFOLDERS)


from checkdeps import DepsChecker  # pylint: disable=wrong-import-position
import licenses  # pylint: disable=wrong-import-position


class _LicensesArgs:
    extra_third_party_dirs = []
    extra_allowed_dirs = []
    exclude_dirs = []
    scan_root = _REPO_PATH
    target_os = None
    gn_out_dir = None
    gn_target = None
    enable_warnings = True
    shipped_only = False
    verbose = False

def _check_licenses(input_api, output_api):
    """Checks third party licenses and returns a list of violations."""
    # NOTE: the licenses check is confused by the fact that we don't actually
    # check out the libraries in buildtools/third_party, so explicitly exclude
    # that folder. See https://crbug.com/1215335 for more info.
    licenses.PRUNE_PATHS.update([
        os.path.join('buildtools', 'third_party'),
        os.path.join('third_party', 'libc++'),
        os.path.join('third_party', 'libc++abi'),
        os.path.join('third_party', 'rust'),
        os.path.join('third_party', 'rust-toolchain'),
        os.path.join('third_party', 'depot_tools'),
        os.path.join('third_party', 'boringssl', 'src', 'third_party'),
        os.path.join('third_party', 'getopt'),
        os.path.join('third_party', 'googleurl', 'src', 'polyfills',
                     'third_party'),
        os.path.join('third_party', 'instrumented_libs', 'binaries'),
        os.path.join('third_party', 'perfetto', 'src', 'protos',
                     'third_party'),
        os.path.join('third_party', 'protobuf', 'third_party'),
        os.path.join('tools', 'clang'),
        os.path.join('tools', 'rust'),
    ])

    if any(s.LocalPath().startswith('third_party') or s.LocalPath() in (
            'DEPS', 'PRESUBMIT.py') for s in input_api.change.AffectedFiles()):
        try:
            _, had_errors = licenses._DiscoverMetadatas(_LicensesArgs())
            if had_errors:
                return [
                    output_api.PresubmitError(
                        "Third party license scan failed. Please check your " +
                        "third_party metadata or licenses.py SPECIAL_CASES."
                    )
                ]
        except Exception as e:
            return [
                output_api.PresubmitError(
                    "License check failed with exception: %s" % e)
            ]
    return []


def _check_generated_infra_files(input_api, output_api):
    files = input_api.UnixLocalPaths()
    if (any(f.endswith('.star') for f in files)
            and all(not f.endswith('.cfg') for f in files)):
        return [
            output_api.PresubmitPromptWarning(
                'You changed .star files, but didn\'t run `lucicfg generate '
                'infra/config/global/main.star`')
        ]

    return []


def _check_deps(input_api, output_api):
    """Checks DEPS rules and returns a list of violations."""
    deps_checker = DepsChecker(input_api.PresubmitLocalPath())
    deps_checker.CheckDirectory(input_api.PresubmitLocalPath())
    deps_results = deps_checker.results_formatter.GetResults()
    return [output_api.PresubmitError(v) for v in deps_results]


# Arguments passed to methods by cpplint.
CpplintArgs = namedtuple("CpplintArgs", "filename clean_lines linenum error")

# A defined error to return to cpplint.
Error = namedtuple("Error", "type message")


def _check_no_regex_matches(regex,
                            cpplint_args,
                            error,
                            include_cpp_files=True):
    """Checks that there are no matches for a specific regex.

  Args:
    regex: The regex to use for matching.
    cpplint_args: The arguments passed to us by cpplint.
    error: The error to return if we find an issue.
    include_cpp_files: Whether C++ files should be checked.
    """
    if not include_cpp_files and not cpplint_args.filename.endswith('.h'):
        return

    line = cpplint_args.clean_lines.elided[cpplint_args.linenum]
    matched = regex.match(line)
    if matched:
        cpplint_args.error(
            cpplint_args.filename, cpplint_args.linenum, error.type, 4,
            f'Error: {error.message} at {matched.group(0).strip()}')


# Matches OSP_CHECK(foo.is_value()) or OSP_DCHECK(foo.is_value())
_RE_PATTERN_VALUE_CHECK = re.compile(
    r'\s*OSP_D?CHECK\([^)]*\.is_value\(\)\);\s*')


def _check_no_value_dchecks(filename, clean_lines, linenum, error):
    """Checks that there are no OSP_DCHECK(foo.is_value()) instances.

    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
    """
    cpplint_args = CpplintArgs(filename, clean_lines, linenum, error)
    error_to_return = Error('runtime/is_value_dchecks',
                            'Unnecessary CHECK for ErrorOr::is_value()')

    _check_no_regex_matches(_RE_PATTERN_VALUE_CHECK, cpplint_args,
                            error_to_return)


# Matches Foo(Foo&&) when not followed by noexcept.
_RE_PATTERN_MOVE_WITHOUT_NOEXCEPT = re.compile(
    r'\s*(?P<classname>\w+)\((?P=classname)&&[^)]*\)\s*(?!noexcept)\s*[{;=]')


def _check_noexcept_on_move(filename, clean_lines, linenum, error):
    """Checks that move constructors are declared with 'noexcept'.

    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
    """
    cpplint_args = CpplintArgs(filename, clean_lines, linenum, error)
    error_to_return = Error('runtime/noexcept',
                            'Move constructor not declared \'noexcept\'')

    # We only check headers as noexcept is meaningful on declarations, not
    # definitions.  This may skip some definitions in .cc files though.
    _check_no_regex_matches(_RE_PATTERN_MOVE_WITHOUT_NOEXCEPT, cpplint_args,
                            error_to_return, False)


# Matches "namespace <foo> {". Since we only check one line at a time, we
# need to call this twice.
_RE_PATTERN_UNNESTED_NAMESPACE = re.compile(
    r'namespace +\w+ +\{')


def _check_unnested_namespaces(filename, clean_lines, linenum, error):
    """Checks that nestable namespaces are nested.

    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
    """

    # If we have a match for a namespace on this line, check the next line for
    # an nestable namespace declaration.
    re = _RE_PATTERN_UNNESTED_NAMESPACE
    if re.match(clean_lines.elided[linenum]):
        cpplint_args = CpplintArgs(filename, clean_lines, linenum + 1, error)
        error_to_return = Error('runtime/nested_namespace',
                                'Please nest namespaces when possible.')
        _check_no_regex_matches(re, cpplint_args, error_to_return)


# Gives additional debug information whenever a linting error occurs.
_CPPLINT_VERBOSE_LEVEL = 4


# - We disable c++11 header checks since Open Screen allows them.
# - We disable whitespace/braces because of various false positives.
# - There are some false positives with 'explicit' checks, but it's useful
#   enough to keep.
# - We add a custom check for 'noexcept' usage.
def _check_change_lints_clean(input_api, output_api):
    """Checks that all '.cc' and '.h' files pass cpplint.py."""
    cpplint = input_api.cpplint
    # Directive that allows access to a protected member _XX of a client class.
    # pylint: disable=protected-access
    cpplint._cpplint_state.ResetErrorCounts()

    cpplint._SetFilters('-build/c++11,-whitespace/braces')
    files = [
        f.AbsoluteLocalPath() for f in input_api.AffectedSourceFiles(None)
    ]

    for file_name in files:
        cpplint.ProcessFile(file_name, _CPPLINT_VERBOSE_LEVEL, [
            _check_noexcept_on_move, _check_no_value_dchecks,
            _check_unnested_namespaces
        ])

    if cpplint._cpplint_state.error_count:
        if input_api.is_committing:
            res_type = output_api.PresubmitError
        else:
            res_type = output_api.PresubmitPromptWarning
        return [res_type('Changelist failed cpplint.py check.')]

    return []


def _check_luci_cfg_lint(input_api, output_api):
    """Check that the luci configs pass the linter."""
    path = os.path.join('infra', 'config', 'global', 'main.star')
    if not input_api.AffectedSourceFiles(
            lambda f: os.path.samefile(f.AbsoluteLocalPath(), path)):
        return []

    result = []
    result.extend(input_api.RunTests([input_api.Command(
        'lucicfg lint',
        [
            'lucicfg' if not input_api.is_windows else 'lucicfg.bat', 'lint',
            path, '--log-level', 'debug' if input_api.verbose else 'warning'
        ],
        {
            'stderr': input_api.subprocess.STDOUT,
            'shell': input_api.is_windows,  # to resolve *.bat
            'cwd': input_api.PresubmitLocalPath(),
        },
        output_api.PresubmitError)]))
    return result


def _check_recipes(input_api, output_api):
    """Runs the recipe simulation tests and lint if any recipe files changed."""
    recipes_dir = os.path.join('infra', 'recipes') + os.sep
    recipes_cfg = os.path.join('infra', 'config', 'recipes.cfg')

    def _is_recipe_file(f):
        path = f.LocalPath()
        return path == recipes_cfg or path.startswith(recipes_dir)

    if not input_api.AffectedFiles(file_filter=_is_recipe_file):
        return []

    recipes_py = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                        'infra', 'recipes', 'recipes.py')
    kwargs = {
        'stderr': input_api.subprocess.STDOUT,
        'cwd': input_api.PresubmitLocalPath(),
    }
    commands = []
    for args in (['test', 'run'], ['lint']):
        commands.append(
            input_api.Command('recipes.py ' + ' '.join(args),
                              [input_api.python3_executable, recipes_py] +
                              args, kwargs, output_api.PresubmitError))
    return input_api.RunTests(commands, parallel=False)


def _common_checks(input_api, output_api):
    """Performs a list of checks that should be used for both presubmission and
       upload validation.
    """
    # PanProjectChecks include:
    #   CheckLongLines (@ 80 cols)
    #   CheckChangeHasNoTabs
    #   CheckChangeHasNoStrayWhitespace
    #   CheckChangeWasUploaded (if committing)
    #   CheckChangeHasDescription
    #   CheckDoNotSubmitInDescription
    #   CheckDoNotSubmitInFiles
    #   CheckLicenses
    results = input_api.canned_checks.PanProjectChecks(input_api,
                                                       output_api,
                                                       owners_check=False)

    # No carriage return characters, files end with one EOL (\n).
    results.extend(
        input_api.canned_checks.CheckChangeHasNoCrAndHasOnlyOneEol(
            input_api, output_api))

    # Ensure code change is gender inclusive.
    results.extend(
        input_api.canned_checks.CheckGenderNeutral(input_api, output_api))

    # Ensure code change to do items uses TODO(bug) or TODO(user) format.
    #  TODO(bug) is generally preferred.
    results.extend(
        input_api.canned_checks.CheckChangeTodoHasOwner(input_api, output_api))

    # Ensure code change passes linter cleanly.
    results.extend(_check_change_lints_clean(input_api, output_api))

    # Ensure code change has already had clang-format ran.
    results.extend(
        input_api.canned_checks.CheckPatchFormatted(input_api,
                                                    output_api,
                                                    bypass_warnings=False))

    # Ensure code change has had GN formatting ran.
    results.extend(
        input_api.canned_checks.CheckGNFormatted(input_api, output_api))

    # Run buildtools/checkdeps on code change.
    results.extend(_check_deps(input_api, output_api))

    # Ensure the LUCI configs pass the linter.
    results.extend(_check_luci_cfg_lint(input_api, output_api))

    # Run the in-repo recipe tests if recipes changed.
    results.extend(_check_recipes(input_api, output_api))

    # Run tools/licenses on code change.
    results.extend(_check_licenses(input_api, output_api))

    results.extend(_check_generated_infra_files(input_api, output_api))

    results.extend(
        input_api.RunTests(
            input_api.canned_checks.CheckVPythonSpec(input_api, output_api)))

    return results


def CheckChangeOnUpload(input_api, output_api):
    """Checks the changelist whenever there is an upload (`git cl upload`)"""
    # We always run the OnCommit checks, as well as some additional checks.
    results = CheckChangeOnCommit(input_api, output_api)
    results.extend(
        input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api))
    return results


def CheckChangeOnCommit(input_api, output_api):
    """Checks the changelist whenever there is commit (`git cl commit`)"""
    return _common_checks(input_api, output_api)