# Copyright 2015 WebAssembly Community Group participants
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import argparse
import difflib
import fnmatch
import glob
import os
import shutil
import stat
import subprocess
import sys
from contextlib import contextmanager
from pathlib import Path

# The C++ standard whose features are required to build Binaryen.
# Keep in sync with CMakeLists.txt CXX_STANDARD
cxx_standard = 17


def parse_args(args):
    usage_str = ("usage: 'python check.py [options]'\n\n"
                 "Runs the Binaryen test suite.")
    parser = argparse.ArgumentParser(description=usage_str)
    parser.add_argument(
        '--torture', dest='torture', action='store_true', default=True,
        help='Chooses whether to run the torture testcases. Default: true.')
    parser.add_argument(
        '--no-torture', dest='torture', action='store_false',
        help='Disables running the torture testcases.')
    parser.add_argument(
        '--abort-on-first-failure', '--fail-fast', dest='abort_on_first_failure',
        action=argparse.BooleanOptionalAction, default=True,
        help=('Specifies whether to halt test suite execution on first test error.'
              ' Default: true.'))
    parser.add_argument(
        '--binaryen-bin', dest='binaryen_bin', default='',
        help=('Specifies the path to the Binaryen executables in the CMake build'
              ' directory. Default: bin/ of current directory (i.e. assume an'
              ' in-tree build).'
              ' If not specified, the environment variable BINARYEN_ROOT= can also'
              ' be used to adjust this.'))
    parser.add_argument(
        '--binaryen-lib', dest='binaryen_lib', default='',
        help=('Specifies a path to where the built Binaryen shared library resides at.'
              ' Default: ./lib relative to bin specified above.'))
    parser.add_argument(
        '--binaryen-root', dest='binaryen_root', default='',
        help=('Specifies a path to the root of the Binaryen repository tree.'
              ' Default: the directory where this file check.py resides.'))
    parser.add_argument(
        '--out-dir', dest='out_dir', default='',
        help=('Specifies a path to the output directory for temp files, which '
              'is also where the test runner changes directory into.'
              ' Default:. out/test under the binaryen root.'))
    parser.add_argument(
        '--valgrind', dest='valgrind', default='',
        help=('Specifies a path to Valgrind tool, which will be used to validate'
              ' execution if specified. (Pass --valgrind=valgrind to search in'
              ' PATH)'))
    parser.add_argument(
        '--valgrind-full-leak-check', dest='valgrind_full_leak_check',
        action='store_true', default=False,
        help=('If specified, all unfreed (but still referenced) pointers at the'
              ' end of execution are considered memory leaks. Default: disabled.'))
    parser.add_argument(
        '--spec-test', action='append', default=[], dest='spec_tests',
        help='Names specific spec tests to run.')
    parser.add_argument(
        'positional_args', metavar='TEST_SUITE', nargs='*',
        help=('Names specific test suites to run. Use --list-suites to see a '
              'list of all test suites'))
    parser.add_argument(
        '--list-suites', action='store_true',
        help='List the test suites that can be run.')
    parser.add_argument(
        '--filter', dest='test_name_filter', default='',
        help=('Specifies a filter. Only tests whose paths contains this '
              'substring will be run'))
    # This option is only for fuzz_opt.py
    # TODO Allow each script to inherit the default set of options and add its
    # own custom options on top of that
    parser.add_argument(
        '--no-auto-initial-contents', dest='auto_initial_contents',
        action='store_false', default=True,
        help='Select important initial contents automaticaly in fuzzer. '
             'Default: disabled.')

    return parser.parse_args(args)


options = parse_args(sys.argv[1:])
requested = options.positional_args
script_dir = os.path.dirname(os.path.abspath(__file__))

num_failures = 0
warnings = []


def warn(text):
    warnings.append(text)
    print('warning:', text, file=sys.stderr)


# setup

# Locate Binaryen build artifacts directory (bin/ by default)
if not options.binaryen_bin:
    if os.environ.get('BINARYEN_ROOT'):
        if os.path.isdir(os.path.join(os.environ.get('BINARYEN_ROOT'), 'bin')):
            options.binaryen_bin = os.path.join(
                os.environ.get('BINARYEN_ROOT'), 'bin')
        else:
            options.binaryen_bin = os.environ.get('BINARYEN_ROOT')
    else:
        options.binaryen_bin = 'bin'

options.binaryen_bin = os.path.normpath(os.path.abspath(options.binaryen_bin))

if not options.binaryen_lib:
    options.binaryen_lib = os.path.join(os.path.dirname(options.binaryen_bin),  'lib')

options.binaryen_lib = os.path.normpath(os.path.abspath(options.binaryen_lib))

options.binaryen_build = os.path.dirname(options.binaryen_bin)

# ensure BINARYEN_ROOT is set up
os.environ['BINARYEN_ROOT'] = os.path.dirname(options.binaryen_bin)

wasm_dis_filenames = ['wasm-dis', 'wasm-dis.exe', 'wasm-dis.js']
if not any(os.path.isfile(os.path.join(options.binaryen_bin, f))
           for f in wasm_dis_filenames):
    warn('Binaryen not found (or has not been successfully built to bin/ ?')

# Locate Binaryen source directory if not specified.
if not options.binaryen_root:
    options.binaryen_root = os.path.dirname(os.path.dirname(script_dir))

options.binaryen_test = os.path.join(options.binaryen_root, 'test')

if not options.out_dir:
    options.out_dir = os.path.join(options.binaryen_root, 'out', 'test')

if not os.path.exists(options.out_dir):
    os.makedirs(options.out_dir)


# Finds the given executable 'program' in PATH.
# Operates like the Unix tool 'which'.
def which(program):
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        paths = [
            # Prefer tools installed using third_party/setup.py
            os.path.join(options.binaryen_root, 'third_party', 'mozjs'),
            os.path.join(options.binaryen_root, 'third_party', 'v8'),
            os.path.join(options.binaryen_root, 'third_party', 'wabt', 'bin'),
        ] + os.environ['PATH'].split(os.pathsep)
        for path in paths:
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
            if '.' not in fname:
                if is_exe(exe_file + '.exe'):
                    return exe_file + '.exe'
                if is_exe(exe_file + '.cmd'):
                    return exe_file + '.cmd'
                if is_exe(exe_file + '.bat'):
                    return exe_file + '.bat'


NATIVECC = (os.environ.get('CC') or which('mingw32-gcc') or
            which('gcc') or which('clang'))
NATIVEXX = (os.environ.get('CXX') or which('mingw32-g++') or
            which('g++') or which('clang++'))
NODEJS = os.environ.get('NODE') or which('node') or which('nodejs')
MOZJS = which('mozjs') or which('spidermonkey')

V8 = os.environ.get('V8') or which('v8') or which('d8')

BINARYEN_INSTALL_DIR = os.path.dirname(options.binaryen_bin)
WASM_OPT = [os.path.join(options.binaryen_bin, 'wasm-opt')]
WASM_AS = [os.path.join(options.binaryen_bin, 'wasm-as')]
WASM_DIS = [os.path.join(options.binaryen_bin, 'wasm-dis')]
WASM2JS = [os.path.join(options.binaryen_bin, 'wasm2js')]
WASM_CTOR_EVAL = [os.path.join(options.binaryen_bin, 'wasm-ctor-eval')]
WASM_SHELL = [os.path.join(options.binaryen_bin, 'wasm-shell')]
WASM_REDUCE = [os.path.join(options.binaryen_bin, 'wasm-reduce')]
WASM_METADCE = [os.path.join(options.binaryen_bin, 'wasm-metadce')]
WASM_EMSCRIPTEN_FINALIZE = [os.path.join(options.binaryen_bin,
                                         'wasm-emscripten-finalize')]
BINARYEN_JS = os.path.join(options.binaryen_bin, 'binaryen_js.js')


def wrap_with_valgrind(cmd):
    # Exit code 97 is arbitrary, used to easily detect when an error occurs that
    # is detected by Valgrind.
    valgrind = [options.valgrind, '--quiet', '--error-exitcode=97']
    if options.valgrind_full_leak_check:
        valgrind += ['--leak-check=full', '--show-leak-kinds=all']
    return valgrind + cmd


if options.valgrind:
    WASM_OPT = wrap_with_valgrind(WASM_OPT)
    WASM_AS = wrap_with_valgrind(WASM_AS)
    WASM_DIS = wrap_with_valgrind(WASM_DIS)
    WASM_SHELL = wrap_with_valgrind(WASM_SHELL)


def in_binaryen(*args):
    return os.path.join(options.binaryen_root, *args)


os.environ['BINARYEN'] = in_binaryen()


def get_platform():
    return {'linux': 'linux',
            'linux2': 'linux',
            'darwin': 'mac',
            'win32': 'windows',
            'cygwin': 'windows'}[sys.platform]


def has_shell_timeout():
    return get_platform() != 'windows' and os.system('timeout 1s pwd') == 0


# Default options to pass to v8. These enable all features.
# See https://github.com/v8/v8/blob/master/src/wasm/wasm-feature-flags.h
V8_OPTS = [
    '--wasm-staging',
    '--experimental-wasm-compilation-hints',
    '--experimental-wasm-stringref',
    '--experimental-wasm-fp16',
    '--experimental-wasm-custom-descriptors',
    '--experimental-wasm-js-interop',
]

# external tools

try:
    if NODEJS is not None:
        subprocess.run([NODEJS, '--version'], check=True, capture_output=True)
except (OSError, subprocess.CalledProcessError):
    NODEJS = None
if NODEJS is None:
    warn('no node found (did not check proper js form)')


# utilities

# removes a file if it exists, using any and all ways of doing so
def delete_from_orbit(filename):
    try:
        os.unlink(filename)
    except OSError:
        pass
    if not os.path.exists(filename):
        return
    try:
        shutil.rmtree(filename, ignore_errors=True)
    except OSError:
        pass
    if not os.path.exists(filename):
        return
    try:
        os.chmod(filename, os.stat(filename).st_mode | stat.S_IWRITE)

        def remove_readonly_and_try_again(func, path, exc_info):
            if not (os.stat(path).st_mode & stat.S_IWRITE):
                os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
                func(path)
            else:
                raise exc_info[1]
        shutil.rmtree(filename, onerror=remove_readonly_and_try_again)
    except OSError:
        pass


def run_process(cmd, check=True, input=None, decode_output=True, *args, **kwargs):
    if input and type(input) is str:
        input = bytes(input, 'utf-8')
    ret = subprocess.run(cmd, *args, check=check, input=input, **kwargs)
    if decode_output and ret.stdout is not None:
        ret.stdout = ret.stdout.decode('utf-8')
    if ret.stderr is not None:
        ret.stderr = ret.stderr.decode('utf-8')
    return ret


def fail_with_error(msg):
    global num_failures
    try:
        num_failures += 1
        raise Exception(msg)
    except Exception as e:
        print(str(e))
        if options.abort_on_first_failure:
            raise


def fail(actual, expected, fromfile='expected'):
    diff_lines = difflib.unified_diff(
        expected.split('\n'), actual.split('\n'),
        fromfile=fromfile, tofile='actual')
    diff_str = ''.join([a.rstrip() + '\n' for a in diff_lines])[:]
    fail_with_error(f'incorrect output, diff:\n\n{diff_str}')


def fail_if_not_identical(actual, expected, fromfile='expected'):
    if expected != actual:
        fail(actual, expected, fromfile=fromfile)


def fail_if_not_contained(actual, expected):
    if expected not in actual:
        fail(actual, expected)


def fail_if_not_identical_to_file(actual, expected_file):
    binary = expected_file.endswith(".wasm") or type(actual) is bytes
    with open(expected_file, 'rb' if binary else 'r') as f:
        fail_if_not_identical(actual, f.read(), fromfile=expected_file)


def get_test_dir(name):
    """Returns the test directory located at BINARYEN_ROOT/test/[name]."""
    return os.path.join(options.binaryen_test, name)


def get_tests(test_dir, extensions=[], recursive=False):
    """Returns the list of test files in a given directory. 'extensions' is a
    list of file extensions. If 'extensions' is empty, returns all files.
    """
    tests = []
    star = '**/*' if recursive else '*'
    if not extensions:
        tests += glob.glob(os.path.join(test_dir, star), recursive=True)
    for ext in extensions:
        tests += glob.glob(os.path.join(test_dir, star + ext), recursive=True)
    if options.test_name_filter:
        tests = fnmatch.filter(tests, options.test_name_filter)
    tests = [item for item in tests if os.path.isfile(item)]
    return sorted(tests)


if options.spec_tests:
    non_existent_tests = [test_name for test_name in options.spec_tests if not os.path.isfile(test_name)]
    if non_existent_tests:
        raise ValueError(f"Supplied test files do not exist: {non_existent_tests}")
    options.spec_tests = [os.path.abspath(t) for t in options.spec_tests]
else:
    options.spec_tests = get_tests(get_test_dir('spec'), ['.wast'], recursive=True)

os.chdir(options.out_dir)

# 11/27/2019: We updated the spec test suite to upstream spec repo. For some
# files that started failing after this update, we added the new files to this
# skip-list and preserved old ones by renaming them to 'old_[FILENAME].wast'
# not to lose coverage. When the cause of the error is fixed or the unsupported
# construct gets support so the new test passes, we can delete the
# corresponding 'old_[FILENAME].wast' file. When you fix the new file and
# delete the old file, make sure you rename the corresponding .wast.log file in
# expected-output/ if any.
# Paths are relative to the test/spec directory
SPEC_TESTS_TO_SKIP = [
    # Requires us to write our own floating point parser
    'const.wast',

    # Unlinkable module accepted
    'linking.wast',

    # Invalid module accepted
    'unreached-invalid.wast',

    # Test invalid
    'elem.wast',

    # Requires wast `either` support
    'threads/thread.wast',

    # Requires better support for multi-threaded tests
    # 'threads/wait_notify.wast',

    # Non-natural alignment is invalid for atomic operations
    'threads/atomic.wast',
]
SPEC_TESTSUITE_PROPOSALS_TO_SKIP = [
    'wide-arithmetic',
]

# Paths are relative to the test/spec/testsuite directory
SPEC_TESTSUITE_TESTS_TO_SKIP = [
    'array_new_elem.wast',  # Failure to parse element segment item abbreviation
    'binary.wast',   # Missing data count section validation
    'comments.wast',  # Issue with carriage returns being treated as newlines
    'const.wast',    # Hex float constant not recognized as out of range
    'conversions.wast',  # Promoted NaN should be canonical
    'data.wast',    # Fail to parse data segment offset abbreviation
    'elem.wast',    # Requires modeling empty declarative segments
    'f32.wast',     # Adding -0 and -nan should give a canonical NaN
    'f64.wast',     # Adding -0 and -nan should give a canonical NaN
    'float_exprs.wast',  # Adding 0 and NaN should give canonical NaN
    'float_misc.wast',   # Rounding wrong on f64.sqrt
    'func.wast',    # Duplicate parameter names not properly rejected
    'global.wast',  # Fail to parse table
    'if.wast',      # Requires more precise unreachable validation
    'imports.wast',  # Requires fixing handling of mutation to imported globals
    'proposals/threads/imports.wast',  # Missing memory type validation on instantiation
    'linking.wast',  # Missing function type validation on instantiation
    'proposals/threads/memory.wast',  # Missing memory type validation on instantiation
    'annotations.wast',  # String annotations IDs should be allowed
    'instance.wast',  # Requires support for table default elements
    'table64.wast',   # Requires validations for table size
    'tag.wast',      # Non-empty tag results allowed by stack switching
    'local_init.wast',  # Requires local validation to respect unnamed blocks
    'ref_func.wast',   # Requires rejecting undeclared functions references
    'ref_is_null.wast',  # Requires support for non-nullable reference types in tables
    'return_call_indirect.wast',  # Requires more precise unreachable validation
    'select.wast',  # Missing validation of type annotation on select
    'table.wast',  # Requires support for table default elements
    'unreached-invalid.wast',  # Requires more precise unreachable validation
    'array.wast',  # Requires support for table default elements
    'br_if.wast',  # Requires more precise branch validation
    'br_on_cast.wast',  # Requires host references to not be externalized i31refs
    'br_on_cast_fail.wast',  # Requires host references to not be externalized i31refs
    'extern.wast',    # Requires ref.host wast constants
    'i31.wast',       # Requires support for table default elements
    'ref_cast.wast',  # Requires host references to not be externalized i31refs
    'ref_test.wast',  # Requires host references to not be externalized i31refs
    'struct.wast',    # Fails to roundtrip unnamed types e.g. `(ref 0)`
    'type-rec.wast',  # Missing function type validation on instantiation
    'type-subtyping.wast',  # ShellExternalInterface::callTable does not handle subtyping
    'memory64.wast',        # Requires validations on the max memory size
    'imports3.wast',  # Requires better checking of exports from the special "spectest" module
    'i16x8_relaxed_q15mulr_s.wast',  # Requires wast `either` support
    'i8x16_relaxed_swizzle.wast',    # Requires wast `either` support
    'relaxed_dot_product.wast',   # Requires wast `either` support
    'relaxed_laneselect.wast',    # Requires wast `either` support
    'relaxed_madd_nmadd.wast',    # Requires wast `either` support
    'relaxed_min_max.wast',       # Requires wast `either` support
    'simd_const.wast',            # Hex float constant not recognized as out of range
    'simd_conversions.wast',      # Promoted NaN should be canonical
    'simd_f32x4.wast',            # Min of 0 and NaN should give a canonical NaN
    'simd_f32x4_arith.wast',      # Adding inf and -inf should give a canonical NaN
    'simd_f32x4_rounding.wast',   # Ceil of NaN should give a canonical NaN
    'simd_f64x2.wast',            # Min of 0 and NaN should give a canonical NaN
    'simd_f64x2_arith.wast',      # Adding inf and -inf should give a canonical NaN
    'simd_f64x2_rounding.wast',   # Ceil of NaN should give a canonical NaN
    'token.wast',                 # Lexer should require spaces between strings and non-paren tokens
]


def _can_run_spec_test(test):
    test = Path(test)
    if 'testsuite' not in test.parts:
        return not any(test.match(f"test/spec/{test_to_skip}") for test_to_skip in SPEC_TESTS_TO_SKIP)

    if any(proposal in test.parts for proposal in SPEC_TESTSUITE_PROPOSALS_TO_SKIP):
        return False

    return not any(Path(test).match(f"test/spec/testsuite/{test_to_skip}") for test_to_skip in SPEC_TESTSUITE_TESTS_TO_SKIP)


options.spec_tests = [t for t in options.spec_tests if _can_run_spec_test(t)]


# check utilities


def binary_format_check(wast, verify_final_result=True, wasm_as_args=['-g'],
                        binary_suffix='.fromBinary', base_name=None, stdout=None):
    # checks we can convert the wast to binary and back

    as_file = f"{base_name}-a.wasm" if base_name is not None else "a.wasm"
    disassembled_file = f"{base_name}-ab.wast" if base_name is not None else "ab.wast"

    print('         (binary format check)', file=stdout)
    cmd = WASM_AS + [wast, '-o', as_file, '-all'] + wasm_as_args
    print('            ', ' '.join(cmd), file=stdout)
    if os.path.exists(as_file):
        os.unlink(as_file)
    subprocess.check_call(cmd, stdout=subprocess.PIPE)
    assert os.path.exists(as_file)

    cmd = WASM_DIS + [as_file, '-o', disassembled_file, '-all']
    print('            ', ' '.join(cmd), file=stdout)
    if os.path.exists(disassembled_file):
        os.unlink(disassembled_file)
    subprocess.check_call(cmd, stdout=subprocess.PIPE)
    assert os.path.exists(disassembled_file)

    # make sure it is a valid wast
    cmd = WASM_OPT + [disassembled_file, '-all', '-q']
    print('            ', ' '.join(cmd), file=stdout)
    subprocess.check_call(cmd, stdout=subprocess.PIPE)

    if verify_final_result:
        actual = open(disassembled_file).read()
        fail_if_not_identical_to_file(actual, wast + binary_suffix)

    return disassembled_file


@contextmanager
def with_pass_debug():
    old_pass_debug = os.environ.get('BINARYEN_PASS_DEBUG')
    os.environ['BINARYEN_PASS_DEBUG'] = '1'
    try:
        yield
    finally:
        if old_pass_debug is not None:
            os.environ['BINARYEN_PASS_DEBUG'] = old_pass_debug
        elif 'BINARYEN_PASS_DEBUG' in os.environ:
            del os.environ['BINARYEN_PASS_DEBUG']


# checks if we are on windows, and if so logs out that a test is being skipped,
# and returns True. This is a central location for all test skipping on
# windows, so that we can easily find which tests are skipped.
def skip_if_on_windows(name):
    if get_platform() == 'windows':
        print(f'skipping test "{name}" on windows')
        return True
    return False


test_suffixes = ['*.wasm', '*.wast', '*.wat']


# return a list of all the tests in the entire test suite
def get_all_tests():
    core_tests = get_tests(get_test_dir('.'), test_suffixes)
    passes_tests = get_tests(get_test_dir('passes'), test_suffixes)
    spec_tests = get_tests(get_test_dir('spec'), test_suffixes)
    wasm2js_tests = get_tests(get_test_dir('wasm2js'), test_suffixes)
    lld_tests = get_tests(get_test_dir('lld'), test_suffixes)
    unit_tests = get_tests(get_test_dir(os.path.join('unit', 'input')), test_suffixes)
    lit_tests = get_tests(get_test_dir('lit'), test_suffixes, recursive=True)

    return core_tests + passes_tests + spec_tests + wasm2js_tests + lld_tests + unit_tests + lit_tests