#!/usr/bin/env python2
# This Python file uses the following encoding: utf-8

'''
Simple test runner. Consider using parallel_test_core.py for faster iteration times.
'''

# XXX Use EM_ALL_ENGINES=1 in the env to test all engines!


from subprocess import Popen, PIPE, STDOUT
import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, re, difflib, webbrowser, hashlib, threading, platform, BaseHTTPServer, SimpleHTTPServer, multiprocessing, functools, stat, string, random, operator, fnmatch, httplib
from urllib import unquote

# Setup

__rootpath__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def path_from_root(*pathelems):
  return os.path.join(__rootpath__, *pathelems)
sys.path += [path_from_root(''), path_from_root('third_party/websockify')]
import tools.shared
from tools.shared import *
from tools.line_endings import check_line_endings

# Sanity check for config

try:
  assert COMPILER_OPTS != None
except:
  raise Exception('Cannot find "COMPILER_OPTS" definition. Is %s set up properly? You may need to copy the template settings file into it.' % EM_CONFIG)

HELP_TEXT = '''
==============================================================================
Running the main part of the test suite. Don't forget to run the other parts!
A recommended order is:

  sanity - tests for first run, etc., modifies ~/.emscripten
  (the main test suite)
  other - tests separate from the main suite
  browser - runs pages in a web browser
  interactive - runs interactive browser tests that need human verification, and could not be automated
  sockets - runs websocket networking tests
  benchmark - run before and after each set of changes before pushing to
              master, verify no regressions

To run one of those parts, do something like

  python tests/runner.py sanity

To run a specific set of tests, you can do things like

  python tests/runner.py asm2

(that runs the asm2 (asm.js, -O2) tests). You can run individual tests with

  python tests/runner.py test_hello_world

Combinations work too, for example

  python tests/runner.py browser.test_sdl_image

In the main test suite, you can run all variations (O0, O1, O2, etc.) of
an individual test with

  python tests/runner.py ALL.test_hello_world

You can run a random set of N tests with a command like

  python tests/runner.py random50

An individual test can be skipped by passing the "skip:" prefix. E.g.

  python tests/runner.py other skip:other.test_cmake

Passing a wildcard allows choosing a subset of tests in a suite, e.g.

  python tests/runner.py browser.test_pthread_*

will run all the pthreads related tests. Wildcards can also be passed in skip,
so

  python tests/runner.py browser skip:browser.test_pthread_*

will run the whole browser suite except for all the pthread tests in it.

Debugging: You can run

  EM_SAVE_DIR=1 python tests/runner.py ALL.test_hello_world

in order to save the test runner directory, in /tmp/emscripten_temp. All files
created by the test will be present there. You can also use EMCC_DEBUG to
further debug the compiler itself, see emcc.
==============================================================================

'''

# Core test runner class, shared between normal tests and benchmarks
checked_sanity = False
test_modes = [
  'default',
  'asm1',
  'asm2',
  'asm3',
  'asm2f',
  'asm2g',
  'asm2i',
  'asm2nn'
]
nondefault_test_modes = [
  'binaryen0',
  'binaryen1',
  'binaryen2',
  'binaryen3',
  'binaryen_native'
]
test_index = 0

use_all_engines = os.environ.get('EM_ALL_ENGINES') # generally js engines are equivalent, testing 1 is enough. set this
                                                   # to force testing on all js engines, good to find js engine bugs

class RunnerCore(unittest.TestCase):
  emcc_args = None
  save_dir = os.environ.get('EM_SAVE_DIR')
  save_JS = 0
  stderr_redirect = STDOUT # This avoids cluttering the test runner output, which is stderr too, with compiler warnings etc.
                           # Change this to None to get stderr reporting, for debugging purposes

  env = {}

  EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS = int(os.getenv('EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS')) if os.getenv('EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS') != None else 0

  temp_files_before_run = []

  def skipme(self): # used by tests we ask on the commandline to be skipped, see right before call to unittest.main
    return self.skip('requested to be skipped')

  def is_emterpreter(self):
    return False

  def is_wasm_backend(self):
    return LLVM_TARGET == WASM_TARGET

  def uses_memory_init_file(self):
    if self.emcc_args is None:
      return None
    elif '--memory-init-file' in self.emcc_args:
      return int(self.emcc_args[self.emcc_args.index('--memory-init-file')+1])
    else:
      # side modules handle memory differently; binaryen puts the memory in the wasm module
      return ('-O2' in self.emcc_args or '-O3' in self.emcc_args or '-Oz' in self.emcc_args) and not (Settings.SIDE_MODULE or Settings.BINARYEN)

  def setUp(self):
    Settings.reset()

    if self.EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS:
      for root, dirnames, filenames in os.walk(TEMP_DIR):
        for dirname in dirnames: self.temp_files_before_run.append(os.path.normpath(os.path.join(root, dirname)))
        for filename in filenames: self.temp_files_before_run.append(os.path.normpath(os.path.join(root, filename)))

    self.banned_js_engines = []
    self.use_all_engines = use_all_engines
    if not self.save_dir:
      dirname = tempfile.mkdtemp(prefix='emscripten_test_' + self.__class__.__name__ + '_', dir=TEMP_DIR)
    else:
      dirname = CANONICAL_TEMP_DIR
    if not os.path.exists(dirname):
      os.makedirs(dirname)
    self.working_dir = dirname
    os.chdir(dirname)

    # Use emscripten root for node module lookup
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    os.environ['NODE_PATH'] = os.path.join(scriptdir, '..', 'node_modules')

    if not self.save_dir:
      self.has_prev_ll = False
      for temp_file in os.listdir(TEMP_DIR):
        if temp_file.endswith('.ll'):
          self.has_prev_ll = True

  def tearDown(self):
    if not self.save_dir:
      # rmtree() fails on Windows if the current working directory is inside the tree.
      os.chdir(os.path.join(self.get_dir(), '..'))
      try_delete(self.get_dir())

      if self.EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS and not os.environ.get('EMCC_DEBUG'):
        temp_files_after_run = []
        for root, dirnames, filenames in os.walk(TEMP_DIR):
          for dirname in dirnames: temp_files_after_run.append(os.path.normpath(os.path.join(root, dirname)))
          for filename in filenames: temp_files_after_run.append(os.path.normpath(os.path.join(root, filename)))

        left_over_files = list(set(temp_files_after_run) - set(self.temp_files_before_run))
        if len(left_over_files) > 0:
          print >> sys.stderr, 'ERROR: After running test, there are ' + str(len(left_over_files)) + ' new temporary files/directories left behind:'
          for f in left_over_files:
            print >> sys.stderr, 'leaked file: ' + f
          raise Exception('Test leaked ' + str(len(left_over_files)) + ' temporary files!')

      # Make sure we don't leave stuff around
      #if not self.has_prev_ll:
      #  for temp_file in os.listdir(TEMP_DIR):
      #    assert not temp_file.endswith('.ll'), temp_file
      #    # TODO assert not temp_file.startswith('emscripten_'), temp_file

  def skip(self, why):
    print >> sys.stderr, '<skipping: %s> ' % why,
    return False

  def get_dir(self):
    return self.working_dir

  def in_dir(self, *pathelems):
    return os.path.join(self.get_dir(), *pathelems)

  def get_stdout_path(self):
    return os.path.join(self.get_dir(), 'stdout')

  def hardcode_arguments(self, filename, args):
    # Hardcode in the arguments, so js is portable without manual commandlinearguments
    if not args: return
    js = open(filename).read()
    open(filename, 'w').write(js.replace('run();', 'run(%s + Module["arguments"]);' % str(args)))

  def prep_ll_run(self, filename, ll_file, force_recompile=False, build_ll_hook=None):
    #force_recompile = force_recompile or os.stat(filename + '.o.ll').st_size > 50000 # if the file is big, recompile just to get ll_opts # Recompiling just for dfe in ll_opts is too costly

    if Building.LLVM_OPTS or force_recompile or build_ll_hook:
      if ll_file.endswith(('.bc', '.o')):
        if ll_file != filename + '.o':
          shutil.copy(ll_file, filename + '.o')
        Building.llvm_dis(filename)
      else:
        shutil.copy(ll_file, filename + '.o.ll')

      Building.ll_opts(filename)
      if build_ll_hook:
        need_post = build_ll_hook(filename)
      Building.llvm_as(filename)
      shutil.move(filename + '.o.ll', filename + '.o.ll.pre') # for comparisons later
      if Building.LLVM_OPTS:
        Building.llvm_opts(filename)
      Building.llvm_dis(filename)
      if build_ll_hook and need_post:
        build_ll_hook(filename)
        Building.llvm_as(filename)
        shutil.move(filename + '.o.ll', filename + '.o.ll.post') # for comparisons later
        Building.llvm_dis(filename)

      Building.llvm_as(filename)
    else:
      if ll_file.endswith('.ll'):
        safe_copy(ll_file, filename + '.o.ll')
        Building.llvm_as(filename)
      else:
        safe_copy(ll_file, filename + '.o')

  # Generate JS from ll, and optionally modify the generated JS with a post_build function. Note
  # that post_build is called on unoptimized JS, so we send it to emcc (otherwise, if run after
  # emcc, it would not apply on the optimized/minified JS)
  def ll_to_js(self, filename, extra_emscripten_args, post_build):
    if type(post_build) in (list, tuple):
      post1, post2 = post_build
    else:
      post1 = post_build
      post2 = None

    emcc_args = self.emcc_args
    if emcc_args is None:
      emcc_args = []

    transform_args = []
    if post1:
      transform_filename = os.path.join(self.get_dir(), 'transform.py')
      transform = open(transform_filename, 'w')
      transform.write('\nimport sys\nsys.path += [%r]\n' % path_from_root(''))
      transform.write(post1)
      transform.write('\nprocess(sys.argv[1])\n')
      transform.close()
      transform_args = ['--js-transform', "%s %s" % (PYTHON, transform_filename)]
    Building.emcc(filename + '.o', Settings.serialize() + emcc_args + transform_args + Building.COMPILER_TEST_OPTS, filename + '.o.js')
    if post2: post2(filename + '.o.js')

  # Build JavaScript code from source code
  def build(self, src, dirname, filename, output_processor=None, main_file=None, additional_files=[], libraries=[], includes=[], build_ll_hook=None, extra_emscripten_args=[], post_build=None, js_outfile=True):

    Building.pick_llvm_opts(3) # pick llvm opts here, so we include changes to Settings in the test case code

    # Copy over necessary files for compiling the source
    if main_file is None:
      f = open(filename, 'w')
      f.write(src)
      f.close()
      final_additional_files = []
      for f in additional_files:
        final_additional_files.append(os.path.join(dirname, os.path.basename(f)))
        shutil.copyfile(f, final_additional_files[-1])
      additional_files = final_additional_files
    else:
      # copy whole directory, and use a specific main .cpp file
      # (rmtree() fails on Windows if the current working directory is inside the tree.)
      if os.getcwd().startswith(os.path.abspath(dirname)):
          os.chdir(os.path.join(dirname, '..'))
      shutil.rmtree(dirname)
      shutil.copytree(src, dirname)
      shutil.move(os.path.join(dirname, main_file), filename)
      # the additional files were copied; alter additional_files to point to their full paths now
      additional_files = map(lambda f: os.path.join(dirname, f), additional_files)
      os.chdir(self.get_dir())

    if build_ll_hook or post_build or extra_emscripten_args:
      # "slow", old path: build to bc, then build to JS

      # C++ => LLVM binary

      for f in [filename] + additional_files:
        try:
          # Make sure we notice if compilation steps failed
          os.remove(f + '.o')
        except:
          pass
        args = [PYTHON, EMCC] + Building.COMPILER_TEST_OPTS + Settings.serialize() + \
               ['-I', dirname, '-I', os.path.join(dirname, 'include')] + \
               map(lambda include: '-I' + include, includes) + \
               ['-c', f, '-o', f + '.o']
        output = subprocess.check_call(args, stderr=self.stderr_redirect if not DEBUG else None)
        assert os.path.exists(f + '.o')

      # Link all files
      if len(additional_files) + len(libraries) > 0:
        shutil.move(filename + '.o', filename + '.o.alone')
        Building.link([filename + '.o.alone'] + map(lambda f: f + '.o', additional_files) + libraries,
                 filename + '.o')
        if not os.path.exists(filename + '.o'):
          print "Failed to link LLVM binaries:\n\n", output
          raise Exception("Linkage error");

      # Finalize
      self.prep_ll_run(filename, filename + '.o', build_ll_hook=build_ll_hook)

      # BC => JS
      self.ll_to_js(filename, extra_emscripten_args, post_build)
    else:
      # "fast", new path: just call emcc and go straight to JS
      all_files = [filename] + additional_files + libraries
      for i in range(len(all_files)):
        if '.' not in all_files[i]:
          shutil.move(all_files[i], all_files[i] + '.bc')
          all_files[i] += '.bc'
      args = [PYTHON, EMCC] + Building.COMPILER_TEST_OPTS + Settings.serialize() + \
             self.emcc_args + \
             ['-I', dirname, '-I', os.path.join(dirname, 'include')] + \
             map(lambda include: '-I' + include, includes) + \
             all_files + \
             ['-o', filename + '.o.js']
      output = subprocess.check_call(args, stderr=self.stderr_redirect if not DEBUG else None)
      if js_outfile:
        assert os.path.exists(filename + '.o.js')

    if output_processor is not None:
      output_processor(open(filename + '.o.js').read())

    if self.emcc_args is not None and js_outfile:
      src = open(filename + '.o.js').read()
      if self.uses_memory_init_file():
        # side memory init file, or an empty one in the js
        assert ('/* memory initializer */' not in src) or ('/* memory initializer */ allocate([]' in src)

  def validate_asmjs(self, err):
    m = re.search("asm.js type error: '(\w+)' is not a (standard|supported) SIMD type", err)
    if m:
      # Bug numbers for missing SIMD types:
      bugs = {
        'Int8x16'  : 1136226,
        'Int16x8'  : 1136226,
        'Uint8x16' : 1244117,
        'Uint16x8' : 1244117,
        'Uint32x4' : 1240796,
        'Float64x2': 1124205,
      }
      simd = m.group(1)
      if simd in bugs:
        print >> sys.stderr, ("\nWARNING: ignoring asm.js type error from {} due to implementation not yet available in SpiderMonkey." +
            " See https://bugzilla.mozilla.org/show_bug.cgi?id={}\n").format(simd, bugs[simd])
        err = err.replace(m.group(0), '')

    if 'uccessfully compiled asm.js code' in err and 'asm.js link error' not in err:
      print >> sys.stderr, "[was asm.js'ified]"
    elif 'asm.js' in err: # if no asm.js error, then not an odin build
      raise Exception("did NOT asm.js'ify: " + err)
    err = '\n'.join(filter(lambda line: 'uccessfully compiled asm.js code' not in line, err.split('\n')))
    return err

  def get_func(self, src, name):
    start = src.index('function ' + name + '(')
    t = start
    n = 0
    while True:
      if src[t] == '{': n += 1
      elif src[t] == '}':
        n -= 1
        if n == 0: return src[start:t+1]
      t += 1
      assert t < len(src)

  def count_funcs(self, javascript_file):
    num_funcs = 0
    start_tok = "// EMSCRIPTEN_START_FUNCS"
    end_tok = "// EMSCRIPTEN_END_FUNCS"
    start_off = 0
    end_off = 0

    with open (javascript_file, 'rt') as fin:
      blob = "".join(fin.readlines())
      start_off = blob.find(start_tok) + len(start_tok)
      end_off = blob.find(end_tok)
      asm_chunk = blob[start_off:end_off]
      num_funcs = asm_chunk.count('function ')

    return num_funcs

  def run_generated_code(self, engine, filename, args=[], check_timeout=True, output_nicerizer=None, assert_returncode=0):
    stdout = os.path.join(self.get_dir(), 'stdout') # use files, as PIPE can get too full and hang us
    stderr = os.path.join(self.get_dir(), 'stderr')
    try:
      cwd = os.getcwd()
    except:
      cwd = None
    os.chdir(self.get_dir())
    assert(check_line_endings(filename) == 0) # Make sure that we produced proper line endings to the .js file we are about to run.
    run_js(filename, engine, args, check_timeout, stdout=open(stdout, 'w'), stderr=open(stderr, 'w'), assert_returncode=assert_returncode)
    if cwd is not None:
      os.chdir(cwd)
    out = open(stdout, 'r').read()
    err = open(stderr, 'r').read()
    if engine == SPIDERMONKEY_ENGINE and Settings.ASM_JS == 1:
      err = self.validate_asmjs(err)
    if output_nicerizer:
      ret = output_nicerizer(out, err)
    else:
      ret = out + err
    assert 'strict warning:' not in ret, 'We should pass all strict mode checks: ' + ret
    return ret

  def build_native(self, filename, args=[]):
    compiler = CLANG if filename.endswith('cpp') else CLANG_CC
    process = Popen([compiler, '-O2', '-fno-math-errno', filename, '-o', filename+'.native'] + args, stdout=PIPE, stderr=self.stderr_redirect)
    output = process.communicate()
    if process.returncode is not 0:
      print >> sys.stderr, "Building native executable with command '%s' failed with a return code %d!" % (' '.join([CLANG, '-O2', filename, '-o', filename+'.native']), process.returncode)
      print "Output: " + output[0]

  def run_native(self, filename, args):
    process = Popen([filename+'.native'] + args, stdout=PIPE);
    output = process.communicate()
    if process.returncode is not 0:
      print >> sys.stderr, "Running native executable with command '%s' failed with a return code %d!" % (' '.join([filename+'.native'] + args), process.returncode)
      print "Output: " + output[0]
    return output[0]

  # Tests that the given two paths are identical, modulo path delimiters. E.g. "C:/foo" is equal to "C:\foo".
  def assertPathsIdentical(self, path1, path2):
    path1 = path1.replace('\\', '/')
    path2 = path2.replace('\\', '/')
    return self.assertIdentical(path1, path2)

  # Tests that the given two multiline text content are identical, modulo line ending differences (\r\n on Windows, \n on Unix).
  def assertTextDataIdentical(self, text1, text2):
    text1 = text1.replace('\r\n', '\n')
    text2 = text2.replace('\r\n', '\n')
    return self.assertIdentical(text1, text2)

  def assertIdentical(self, values, y):
    if type(values) not in [list, tuple]: values = [values]
    for x in values:
      if x == y: return # success
    raise Exception("Expected to have '%s' == '%s', diff:\n\n%s" % (
      limit_size(values[0]), limit_size(y),
      limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(x.split('\n'), y.split('\n'), fromfile='expected', tofile='actual')]))
    ))

  def assertTextDataContained(self, text1, text2):
    text1 = text1.replace('\r\n', '\n')
    text2 = text2.replace('\r\n', '\n')
    return self.assertContained(text1, text2)

  def assertContained(self, values, string, additional_info=''):
    if type(values) not in [list, tuple]: values = [values]
    for value in values:
      if type(value) is unicode: string = string.decode('UTF-8') # If we have any non-ASCII chars in the expected string, treat the test string from ASCII as UTF8 as well.
      if type(string) is not str and type(string) is not unicode: string = string()
      if value in string: return # success
    raise Exception("Expected to find '%s' in '%s', diff:\n\n%s\n%s" % (
      limit_size(values[0]), limit_size(string),
      limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(values[0].split('\n'), string.split('\n'), fromfile='expected', tofile='actual')])),
      additional_info
    ))

  def assertNotContained(self, value, string):
    if type(value) is not str: value = value() # lazy loading
    if type(string) is not str: string = string()
    if value in string:
      raise Exception("Expected to NOT find '%s' in '%s', diff:\n\n%s" % (
        limit_size(value), limit_size(string),
        limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(value.split('\n'), string.split('\n'), fromfile='expected', tofile='actual')]))
      ))

  library_cache = {}

  def get_build_dir(self):
    ret = os.path.join(self.get_dir(), 'building')
    if not os.path.exists(ret):
      os.makedirs(ret)
    return ret

  def get_library(self, name, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args='help', cache=True, env_init={}, cache_name_extra='', native=False):
    if make_args == 'help':
      make_args = ['-j', str(multiprocessing.cpu_count())]

    build_dir = self.get_build_dir()
    output_dir = self.get_dir()

    cache_name = name + ','.join(filter(lambda opt: len(opt) < 10, Building.COMPILER_TEST_OPTS)) + '_' + hashlib.md5(str(Building.COMPILER_TEST_OPTS)).hexdigest() + cache_name_extra

    valid_chars = "_%s%s" % (string.ascii_letters, string.digits)
    cache_name = ''.join([(c if c in valid_chars else '_') for c in cache_name])

    if self.library_cache is not None:
      if cache and self.library_cache.get(cache_name):
        print >> sys.stderr,  '<load %s from cache> ' % cache_name
        generated_libs = []
        for basename, contents in self.library_cache[cache_name]:
          bc_file = os.path.join(build_dir, cache_name + '_' +  basename)
          f = open(bc_file, 'wb')
          f.write(contents)
          f.close()
          generated_libs.append(bc_file)
        return generated_libs

    print >> sys.stderr, '<building and saving %s into cache> ' % cache_name

    return Building.build_library(name, build_dir, output_dir, generated_libs, configure, configure_args, make, make_args, self.library_cache, cache_name,
                                  copy_project=True, env_init=env_init, native=native)

  def clear(self, in_curr=False):
    for name in os.listdir(self.get_dir()):
      try_delete(os.path.join(self.get_dir(), name) if not in_curr else name)
    emcc_debug = os.environ.get('EMCC_DEBUG')
    if emcc_debug and not in_curr and EMSCRIPTEN_TEMP_DIR:
      for name in os.listdir(EMSCRIPTEN_TEMP_DIR):
        try_delete(os.path.join(EMSCRIPTEN_TEMP_DIR, name))

  # Shared test code between main suite and others

  def setup_runtimelink_test(self):
    header = r'''
      struct point
      {
        int x, y;
      };

    '''
    open(os.path.join(self.get_dir(), 'header.h'), 'w').write(header)

    supp = r'''
      #include <stdio.h>
      #include "header.h"

      extern void mainFunc(int x);
      extern int mainInt;

      void suppFunc(struct point &p) {
        printf("supp: %d,%d\n", p.x, p.y);
        mainFunc(p.x+p.y);
        printf("supp see: %d\n", mainInt);
      }

      int suppInt = 76;
    '''
    supp_name = os.path.join(self.get_dir(), 'supp.cpp')
    open(supp_name, 'w').write(supp)

    main = r'''
      #include <stdio.h>
      #include "header.h"

      extern void suppFunc(struct point &p);
      extern int suppInt;

      void mainFunc(int x) {
        printf("main: %d\n", x);
      }

      int mainInt = 543;

      int main( int argc, const char *argv[] ) {
        struct point p = { 54, 2 };
        suppFunc(p);
        printf("main see: %d\nok.\n", suppInt);
        #ifdef BROWSER
          int result = suppInt;
          REPORT_RESULT();
        #endif
        return 0;
      }
    '''
    return (main, supp)

  def filtered_js_engines(self, js_engines=None):
    if js_engines is None:
      js_engines = JS_ENGINES
    for engine in js_engines: assert type(engine) == list
    for engine in self.banned_js_engines: assert type(engine) == list
    js_engines = filter(lambda engine: engine[0] not in map(lambda engine: engine[0], self.banned_js_engines), js_engines)
    if 'BINARYEN_METHOD="native-wasm"' in self.emcc_args:
      # when testing native wasm support, must use a vm with support
      js_engines = filter(lambda engine: engine == SPIDERMONKEY_ENGINE or engine == V8_ENGINE, js_engines)
    return js_engines

  def do_run_from_file(self, src, expected_output, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]):
    self.do_run(open(src).read(), open(expected_output).read(),
                args, output_nicerizer, output_processor, no_build, main_file,
                additional_files, js_engines, post_build, basename, libraries,
                includes, force_c, build_ll_hook, extra_emscripten_args)

  ## Does a complete test - builds, runs, checks output, etc.
  def do_run(self, src, expected_output, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[], assert_returncode=None):
    if Settings.ASYNCIFY == 1 and self.is_wasm_backend():
      return self.skip("wasm backend doesn't support ASYNCIFY yet")
    if force_c or (main_file is not None and main_file[-2:]) == '.c':
      basename = 'src.c'
      Building.COMPILER = to_cc(Building.COMPILER)

    dirname = self.get_dir()
    filename = os.path.join(dirname, basename)
    if not no_build:
      self.build(src, dirname, filename, main_file=main_file, additional_files=additional_files, libraries=libraries, includes=includes,
                 build_ll_hook=build_ll_hook, extra_emscripten_args=extra_emscripten_args, post_build=post_build)

    # Run in both JavaScript engines, if optimizing - significant differences there (typed arrays)
    js_engines = self.filtered_js_engines(js_engines)
    if len(js_engines) == 0: return self.skip('No JS engine present to run this test with. Check %s and the paths therein.' % EM_CONFIG)
    if len(js_engines) > 1 and not self.use_all_engines:
      if SPIDERMONKEY_ENGINE in js_engines: # make sure to get asm.js validation checks, using sm
        js_engines = [SPIDERMONKEY_ENGINE]
      else:
        js_engines = js_engines[:1]
    for engine in js_engines:
      #print 'test in', engine
      js_output = self.run_generated_code(engine, filename + '.o.js', args, output_nicerizer=output_nicerizer, assert_returncode=assert_returncode)
      try:
        self.assertContained(expected_output, js_output.replace('\r\n', '\n'))
        self.assertNotContained('ERROR', js_output)
      except Exception, e:
        print '(test did not pass in JS engine: %s)' % engine
        raise e

    #shutil.rmtree(dirname) # TODO: leave no trace in memory. But for now nice for debugging

    if self.save_JS:
      global test_index
      self.hardcode_arguments(filename + '.o.js', args)
      shutil.copyfile(filename + '.o.js', os.path.join(TEMP_DIR, str(test_index) + '.js'))
      test_index += 1

  # No building - just process an existing .ll file (or .bc, which we turn into .ll)
  def do_ll_run(self, ll_file, expected_output=None, args=[], js_engines=None, output_nicerizer=None, post_build=None, force_recompile=False, build_ll_hook=None, extra_emscripten_args=[], assert_returncode=None):
    filename = os.path.join(self.get_dir(), 'src.cpp')

    self.prep_ll_run(filename, ll_file, force_recompile, build_ll_hook)

    self.ll_to_js(filename, extra_emscripten_args, post_build)

    self.do_run(None,
                expected_output,
                args,
                no_build=True,
                js_engines=js_engines,
                output_nicerizer=output_nicerizer,
                post_build=None,
                assert_returncode=assert_returncode) # post_build was already done in ll_to_js, this do_run call is just to test the output


# Run a server and a web page. When a test runs, we tell the server about it,
# which tells the web page, which then opens a window with the test. Doing
# it this way then allows the page to close() itself when done.
def harness_server_func(q):
  class TestServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    def do_GET(s):
      s.send_response(200)
      s.send_header("Content-type", "text/html")
      s.end_headers()
      if s.path == '/run_harness':
        s.wfile.write(open(path_from_root('tests', 'browser_harness.html')).read())
      else:
        result = 'False'
        if not q.empty():
          result = q.get()
        s.wfile.write(result)
      s.wfile.close()
    def log_request(code=0, size=0):
      # don't log; too noisy
      pass
  httpd = BaseHTTPServer.HTTPServer(('localhost', 9999), TestServerHandler)
  httpd.serve_forever() # test runner will kill us

def server_func(dir, q):
  class TestServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def do_GET(self):
      if 'report_' in self.path:
        print '[server response:', self.path, ']'
        q.put(self.path)
        # Send a default OK response to the browser.
        self.send_response(200)
        self.send_header("Content-type", "text/plain")
        self.send_header('Cache-Control','no-cache, must-revalidate')
        self.send_header('Connection','close')
        self.send_header('Expires','-1')
        self.end_headers()
        self.wfile.write('OK')
      else:
        # Use SimpleHTTPServer default file serving operation for GET.
        SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)

    def log_request(code=0, size=0):
      # don't log; too noisy
      pass
  os.chdir(dir)
  httpd = BaseHTTPServer.HTTPServer(('localhost', 8888), TestServerHandler)
  httpd.serve_forever() # test runner will kill us

class BrowserCore(RunnerCore):
  def __init__(self, *args, **kwargs):
    super(BrowserCore, self).__init__(*args, **kwargs)

  @classmethod
  def setUpClass(self):
    super(BrowserCore, self).setUpClass()
    self.browser_timeout = 30
    self.harness_queue = multiprocessing.Queue()
    self.harness_server = multiprocessing.Process(target=harness_server_func, args=(self.harness_queue,))
    self.harness_server.start()
    print '[Browser harness server on process %d]' % self.harness_server.pid
    webbrowser.open_new('http://localhost:9999/run_harness')

  @classmethod
  def tearDownClass(self):
    super(BrowserCore, self).tearDownClass()
    self.harness_server.terminate()
    print '[Browser harness server terminated]'
    # On Windows, shutil.rmtree() in tearDown() raises this exception if we do not wait a bit:
    # WindowsError: [Error 32] The process cannot access the file because it is being used by another process.
    time.sleep(0.1)

  def run_browser(self, html_file, message, expectedResult=None, timeout=None):
    print '[browser launch:', html_file, ']'
    if expectedResult is not None:
      try:
        queue = multiprocessing.Queue()
        server = multiprocessing.Process(target=functools.partial(server_func, self.get_dir()), args=(queue,))
        server.start()
        # Starting the web page server above is an asynchronous procedure, so before we tell the browser below to navigate to
        # the test page, we need to know that the server has started up and is ready to process the site navigation.
        # Therefore block until we can make a connection to the server.
        for i in range(10):
          httpconn = httplib.HTTPConnection('localhost:8888', timeout=1)
          try:
            httpconn.connect()
            httpconn.close()
            break
          except:
            time.sleep(1)
        else:
          raise Exception('[Test harness server failed to start up in a timely manner]')
        self.harness_queue.put('http://localhost:8888/' + html_file)
        output = '[no http server activity]'
        start = time.time()
        if timeout is None: timeout = self.browser_timeout
        while time.time() - start < timeout:
          if not queue.empty():
            output = queue.get()
            break
          time.sleep(0.1)
        if output.startswith('/report_result?skipped:'):
          self.skip(unquote(output[len('/report_result?skipped:'):]).strip())
        else:
          self.assertIdentical(expectedResult, output)
      finally:
        server.terminate()
        time.sleep(0.1) # see comment about Windows above
    else:
      webbrowser.open_new(os.path.abspath(html_file))
      print 'A web browser window should have opened a page containing the results of a part of this test.'
      print 'You need to manually look at the page to see that it works ok: ' + message
      print '(sleeping for a bit to keep the directory alive for the web browser..)'
      time.sleep(5)
      print '(moving on..)'

  def with_report_result(self, code):
    return r'''
#ifdef __EMSCRIPTEN__
  #include <emscripten.h>
  #define REPORT_RESULT_INTERNAL(sync) \
    EM_ASM_({ \
      var xhr = new XMLHttpRequest(); \
      xhr.open('GET', 'http://localhost:8888/report_result?' + $0, !$1); \
      xhr.send(); \
      setTimeout(function() { window.close() }, 1000); \
    }, result, sync);
  #define REPORT_RESULT() REPORT_RESULT_INTERNAL(0)
#endif
''' + code

  def reftest(self, expected):
    # make sure the pngs used here have no color correction, using e.g.
    #   pngcrush -rem gAMA -rem cHRM -rem iCCP -rem sRGB infile outfile
    basename = os.path.basename(expected)
    shutil.copyfile(expected, os.path.join(self.get_dir(), basename))
    open(os.path.join(self.get_dir(), 'reftest.js'), 'w').write('''
      var Module = eval('Module');
      function doReftest() {
        if (doReftest.done) return;
        doReftest.done = true;
        var img = new Image();
        img.onload = function() {
          assert(img.width == Module.canvas.width, 'Invalid width: ' + Module.canvas.width + ', should be ' + img.width);
          assert(img.height == Module.canvas.height, 'Invalid height: ' + Module.canvas.height + ', should be ' + img.height);

          var canvas = document.createElement('canvas');
          canvas.width = img.width;
          canvas.height = img.height;
          var ctx = canvas.getContext('2d');
          ctx.drawImage(img, 0, 0);
          var expected = ctx.getImageData(0, 0, img.width, img.height).data;

          var actualUrl = Module.canvas.toDataURL();
          var actualImage = new Image();
          actualImage.onload = function() {
            /*
            document.body.appendChild(img); // for comparisons
            var div = document.createElement('div');
            div.innerHTML = '^=expected, v=actual';
            document.body.appendChild(div);
            document.body.appendChild(actualImage); // to grab it for creating the test reference
            */

            var actualCanvas = document.createElement('canvas');
            actualCanvas.width = actualImage.width;
            actualCanvas.height = actualImage.height;
            var actualCtx = actualCanvas.getContext('2d');
            actualCtx.drawImage(actualImage, 0, 0);
            var actual = actualCtx.getImageData(0, 0, actualImage.width, actualImage.height).data;

            var total = 0;
            var width = img.width;
            var height = img.height;
            for (var x = 0; x < width; x++) {
              for (var y = 0; y < height; y++) {
                total += Math.abs(expected[y*width*4 + x*4 + 0] - actual[y*width*4 + x*4 + 0]);
                total += Math.abs(expected[y*width*4 + x*4 + 1] - actual[y*width*4 + x*4 + 1]);
                total += Math.abs(expected[y*width*4 + x*4 + 2] - actual[y*width*4 + x*4 + 2]);
              }
            }
            var wrong = Math.floor(total / (img.width*img.height*3)); // floor, to allow some margin of error for antialiasing

            xhr = new XMLHttpRequest();
            xhr.open('GET', 'http://localhost:8888/report_result?' + wrong);
            xhr.send();
            setTimeout(function() { window.close() }, 1000);
          };
          actualImage.src = actualUrl;
        }
        img.src = '%s';
      };
      Module['postRun'] = doReftest;

      if (typeof WebGLClient !== 'undefined') {
        // trigger reftest from RAF as well, needed for workers where there is no pre|postRun on the main thread
        var realRAF = window.requestAnimationFrame;
        window.requestAnimationFrame = function(func) {
          realRAF(function() {
            func();
            realRAF(doReftest);
          });
        };

        // trigger reftest from canvas render too, for workers not doing GL
        var realWOM = worker.onmessage;
        worker.onmessage = function(event) {
          realWOM(event);
          if (event.data.target === 'canvas' && event.data.op === 'render') {
            realRAF(doReftest);
          }
        };
      }

''' % basename)

  def btest(self, filename, expected=None, reference=None, force_c=False, reference_slack=0, manual_reference=False, post_build=None,
      args=[], outfile='test.html', message='.', also_proxied=False, url_suffix='', timeout=None): # TODO: use in all other tests
    # if we are provided the source and not a path, use that
    filename_is_src = '\n' in filename
    src = filename if filename_is_src else ''
    filepath = path_from_root('tests', filename) if not filename_is_src else ('main.c' if force_c else 'main.cpp')
    temp_filepath = os.path.join(self.get_dir(), os.path.basename(filepath))
    original_args = args[:]
    if filename_is_src:
      with open(temp_filepath, 'w') as f: f.write(src)
    if not reference:
      if not src:
        with open(filepath) as f: src = f.read()
      with open(temp_filepath, 'w') as f: f.write(self.with_report_result(src))
    else:
      self.reference = reference
      expected = [str(i) for i in range(0, reference_slack+1)]
      shutil.copyfile(filepath, temp_filepath)
      self.reftest(path_from_root('tests', reference))
      if not manual_reference:
        args = args + ['--pre-js', 'reftest.js', '-s', 'GL_TESTING=1']
    all_args = [PYTHON, EMCC, '-s', 'IN_TEST_HARNESS=1', temp_filepath, '-o', outfile] + args
    #print 'all args:', all_args
    try_delete(outfile)
    Popen(all_args).communicate()
    assert os.path.exists(outfile)
    if post_build: post_build()
    if type(expected) is str: expected = [expected]
    self.run_browser(outfile + url_suffix, message, ['/report_result?' + e for e in expected], timeout=timeout)
    if also_proxied:
      print 'proxied...'
      # save non-proxied
      if not os.path.exists('normal'):
        os.mkdir('normal')
      shutil.copyfile('test.html', os.path.join('normal', 'test.html'))
      shutil.copyfile('test.js', os.path.join('normal', 'test.js'))
      if reference:
        assert not manual_reference
        manual_reference = True
        assert not post_build
        post_build = self.post_manual_reftest
      # run proxied
      self.btest(filename, expected, reference, force_c, reference_slack, manual_reference, post_build, original_args + ['--proxy-to-worker', '-s', 'GL_TESTING=1'], outfile, message, timeout=timeout)

###################################################################################################

def get_zlib_library(runner_core):
  if WINDOWS:
    return runner_core.get_library('zlib', os.path.join('libz.a'), configure=[path_from_root('emconfigure.bat')], configure_args=['cmake', '.', '-DBUILD_SHARED_LIBS=OFF'], make=['mingw32-make'], make_args=[])
  else:
    return runner_core.get_library('zlib', os.path.join('libz.a'), make_args=['libz.a'])

# Both test_core and test_other access the Bullet library, share the access here to avoid duplication.
def get_bullet_library(runner_core, use_cmake):
  if use_cmake:
    configure_commands = ['cmake', '.']
    configure_args = ['-DBUILD_DEMOS=OFF', '-DBUILD_EXTRAS=OFF', '-DUSE_GLUT=OFF']
    # Depending on whether 'configure' or 'cmake' is used to build, Bullet places output files in different directory structures.
    generated_libs = [os.path.join('src', 'BulletDynamics', 'libBulletDynamics.a'),
                      os.path.join('src', 'BulletCollision', 'libBulletCollision.a'),
                      os.path.join('src', 'LinearMath', 'libLinearMath.a')]
  else:
    configure_commands = ['sh', './configure']
    # Force a nondefault --host= so that the configure script will interpret that we are doing cross-compilation
    # and skip attempting to run the generated executable with './a.out', which would fail since we are building a .js file.
    configure_args = ['--host=i686-pc-linux-gnu', '--disable-demos','--disable-dependency-tracking']
    generated_libs = [os.path.join('src', '.libs', 'libBulletDynamics.a'),
                      os.path.join('src', '.libs', 'libBulletCollision.a'),
                      os.path.join('src', '.libs', 'libLinearMath.a')]

  return runner_core.get_library('bullet', generated_libs, configure=configure_commands, configure_args=configure_args, cache_name_extra=configure_commands[0])

if __name__ == '__main__':
  if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
    print HELP_TEXT
    sys.exit(0)

  # If no tests were specified, run the core suite
  if len(sys.argv) == 1:
    sys.argv = [sys.argv[0]] + map(lambda mode: mode, test_modes)
    print HELP_TEXT
    time.sleep(2)

  if use_all_engines:
    print '(using ALL js engines)'
  else:
    logging.warning('use EM_ALL_ENGINES=1 in the env to run against all JS engines, which is slower but provides more coverage')

  # Sanity checks
  total_engines = len(JS_ENGINES)
  JS_ENGINES = filter(jsrun.check_engine, JS_ENGINES)
  if len(JS_ENGINES) == 0:
    print 'WARNING: None of the JS engines in JS_ENGINES appears to work.'
  elif len(JS_ENGINES) < total_engines:
    print 'WARNING: Not all the JS engines in JS_ENGINES appears to work, ignoring those.'

  # Create a list of modules to load tests from
  modules = []
  for filename in glob.glob(os.path.join(os.path.dirname(__file__), 'test*.py')):
    module_dir, module_file = os.path.split(filename)
    module_name, module_ext = os.path.splitext(module_file)
    __import__(module_name)
    modules.append(sys.modules[module_name])

  # Extract the JS engine override from the arguments (used by benchmarks)
  for i in range(1, len(sys.argv)):
    arg = sys.argv[i]
    if arg.isupper():
      print 'Interpreting all capital argument "%s" as JS_ENGINE override' % arg
      Building.JS_ENGINE_OVERRIDE = eval(arg)
      sys.argv[i] = None
  sys.argv = filter(lambda arg: arg is not None, sys.argv)

  # If an argument comes in as test_*, treat it as a test of the default suite
  sys.argv = map(lambda arg: arg if not arg.startswith('test_') else 'default.' + arg, sys.argv)

  # If a test (e.g. test_html) is specified as ALL.test_html, add an entry for each test_mode
  new_args = [sys.argv[0]]
  for i in range(1, len(sys.argv)):
    arg = sys.argv[i]
    if arg.startswith('ALL.'):
      ignore, test = arg.split('.')
      print 'Running all test modes on test "%s"' % test
      new_args += map(lambda mode: mode+'.'+test, test_modes)
    else:
      new_args += [arg]
  sys.argv = new_args

  # Create a list of all known tests so that we can choose from them based on a wildcard search
  all_tests = []
  suites = test_modes + nondefault_test_modes + \
           ['other', 'browser', 'sanity', 'sockets', 'interactive']
  for m in modules:
    for s in suites:
      if hasattr(m, s):
        tests = filter(lambda t: t.startswith('test_'), dir(getattr(m, s)))
        all_tests += map(lambda t: s + '.' + t, tests)

  # Process wildcards, e.g. "browser.test_pthread_*" should expand to list all pthread tests
  new_args = [sys.argv[0]]
  for i in range(1, len(sys.argv)):
    arg = sys.argv[i]
    if '*' in arg:
      if arg.startswith('skip:'):
        arg = arg[5:]
        matching_tests = fnmatch.filter(all_tests, arg)
        new_args += map(lambda t: 'skip:' + t, matching_tests)
      else:
        new_args += fnmatch.filter(all_tests, arg)
    else:
      new_args += [arg]
  if len(new_args) == 1 and len(sys.argv) > 1:
    print 'No tests found to run in set ' + str(sys.argv[1:])
    sys.exit(0)
  sys.argv = new_args

  # Skip requested tests
  for i in range(len(sys.argv)):
    arg = sys.argv[i]
    if arg.startswith('skip:'):
      which = arg.split('skip:')[1]
      if which.startswith('ALL.'):
        ignore, test = which.split('.')
        which = map(lambda mode: mode+'.'+test, test_modes)
      else:
        which = [which]

      print >> sys.stderr, ','.join(which)
      for test in which:
        print >> sys.stderr, 'will skip "%s"' % test
        for m in modules:
          try:
            exec('m.' + test + ' = RunnerCore("skipme")')
            break
          except:
            pass
      sys.argv[i] = None
  sys.argv = filter(lambda arg: arg is not None, sys.argv)

  # If we were asked to run random tests, do that
  first = sys.argv[1]
  if first.startswith('random'):
    num = 1
    first = first[6:]
    base_module = 'default'
    relevant_modes = test_modes
    if len(first) > 0:
      if first.startswith('other'):
        base_module = 'other'
        relevant_modes = ['other']
        first = first.replace('other', '')
      elif first.startswith('browser'):
        base_module = 'browser'
        relevant_modes = ['browser']
        first = first.replace('browser', '')
      num = int(first)
    for m in modules:
      if hasattr(m, base_module):
        sys.argv = [sys.argv[0]]
        tests = filter(lambda t: t.startswith('test_'), dir(getattr(m, base_module)))
        print
        chosen = set()
        while len(chosen) < num:
          test = random.choice(tests)
          mode = random.choice(relevant_modes)
          new_test = mode + '.' + test
          before = len(chosen)
          chosen.add(new_test)
          if len(chosen) > before:
            print '* ' + new_test
          else:
            # we may have hit the limit
            if len(chosen) == len(tests)*len(relevant_modes):
              print '(all possible tests chosen! %d = %d*%d)' % (len(chosen), len(tests), len(relevant_modes))
              break
        sys.argv += list(chosen)
        std = 0.5/math.sqrt(num)
        print
        print 'running those %d randomly-selected tests. if they all pass, then there is a greater than 95%% chance that at least %.2f%% of the test suite will pass' % (num, 100.0-100.0*std)
        print

        import atexit
        def show():
          print 'if all tests passed then there is a greater than 95%% chance that at least %.2f%% of the test suite will pass' % (100.0-100.0*std)
        atexit.register(show)

  # Filter and load tests from the discovered modules
  loader = unittest.TestLoader()
  names = set(sys.argv[1:])
  suites = []
  for m in modules:
    mnames = []
    for name in list(names):
      try:
        operator.attrgetter(name)(m)
        mnames.append(name)
        names.remove(name)
      except AttributeError:
        pass
    if len(mnames) > 0:
      suites.append((m.__name__, loader.loadTestsFromNames(sorted(mnames), m)))

  resultMessages = []
  numFailures = 0

  if len(names) > 0:
    print 'WARNING: could not find the following tests: ' + ' '.join(names)
    numFailures += len(names)
    resultMessages.append('Could not find %s tests' % (len(names),))

  print 'Test suites:'
  print [s[0] for s in suites]
  # Run the discovered tests
  testRunner = unittest.TextTestRunner(verbosity=2)
  for mod_name, suite in suites:
    print 'Running %s: (%s tests)' % (mod_name, suite.countTestCases())
    res = testRunner.run(suite)
    msg = '%s: %s run, %s errors, %s failures, %s skipped' % (mod_name,
        res.testsRun, len(res.errors), len(res.failures), len(res.skipped)
    )
    numFailures += len(res.errors) + len(res.failures)
    resultMessages.append(msg)

  if len(resultMessages) > 1:
    print '===================='
    print
    print 'TEST SUMMARY'
    for msg in resultMessages:
      print '    ' + msg

  # Return the number of failures as the process exit code for automating success/failure reporting.
  exitcode = min(numFailures, 255)
  sys.exit(exitcode)