#!/usr/bin/env ruby

# Copyright (C) 2013-2025 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer. 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

require 'fileutils'
require 'getoptlong'
require 'ostruct'
require 'pathname'
require 'rbconfig'
require 'set'
require 'tempfile'
require 'uri'
require 'yaml'

require_relative "webkitruby/jsc-stress-test/test-result-evaluator"
require_relative "webkitruby/jsc-stress-test/executor"

module URI
    class SSH < Generic
        DEFAULT_PORT = 22
    end
    unless defined?(self.register_scheme)
      def self.register_scheme(scheme, klass)
        @@schemes[scheme] = klass
      end
    end
    register_scheme 'SSH', SSH
end

class String
    def scrub
        encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
    end
end

Signal.trap("TERM") {
    # Occasionally, the jscore-test step fails to produce any output for 10' at
    # which point it gets killed by the buildbot code. Try to diagnose.
    puts("Received SIGTERM")
    puts(Thread.current.backtrace)
}

RemoteHost = Struct.new(:name, :user, :host, :port, :remoteDirectory, :identity_file_path)

THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
JETSTREAM2_PATH = WEBKIT_PATH + "PerformanceTests/JetStream2"
CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"

HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
STATUS_FILE = "test_results"
STATUS_FILE_PASS = "P"
STATUS_FILE_FAIL = "F"

# There are all random and adequately large to be unlikely to appear
# in practice, except as used in this file.
PARALLEL_REMOTE_WRAPPER_MARK_BEGIN = "5d65329bd1a3"
PARALLEL_REMOTE_WRAPPER_MARK_END = "a9aea5c3b843"
PARALLEL_REMOTE_STATE_LOST_MARKER = "709fb7a77c45231918eb118a"
ITERATION_LIMITS = OpenStruct.new(:infraIterationsFloor => 3,
                                  :iterationsCeiling => 50)

REMOTE_TIMEOUT = 120
SSH_OPTIONS_DEFAULT = [
    "-o",
    "NoHostAuthenticationForLocalhost=yes",
    "-o",
    "ServerAliveInterval=30",
]

begin
    require 'shellwords'
rescue Exception => e
    $stderr.puts "Warning: did not find shellwords, not running any tests."
    exit 0
end

$canRunDisplayProfilerOutput = false

begin
    require 'rubygems'
    require 'json'
    require 'highline'
    $canRunDisplayProfilerOutput = true
rescue Exception => e
    $stderr.puts "Warning: did not find json or highline; some features will be disabled."
    $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
    $stderr.puts "Error: #{e.inspect}"
end

def printCommandArray(*cmd)
    begin
        commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
    rescue
        commandArray = cmd.join(' ')
    end
    $stderr.puts ">> #{commandArray}"
end

class CommandExecutionFailed < Exception
end

def mysys(commandArray, options={})
    printCommandArray(commandArray) if $verbosity >= 1
    successful = system(*commandArray)
    status = $?
    if successful or options[:ignoreFailure]
        return
    end
    raise CommandExecutionFailed, "Command failed: #{status.inspect}"
end

def escapeAll(array)
    array.map {
        | v |
        raise "Detected a non-string in #{inspect}" unless v.is_a? String
        Shellwords.shellescape(v)
    }.join(' ')
end

$jscPath = nil
$doNotMessWithVMPath = false
$cloop = false
$jitTests = true
$memoryLimited = false
$outputDir = Pathname.new("results")
$verbosity = 0
$bundle = nil
$tarball = false
$tarFileName = "payload.tar.gz"
$copyVM = false
$testRunnerType = nil
$testWriter = "default"
$remoteHosts = []
$architecture = nil
$forceArchitecture = nil
$hostOS = nil
$model = nil
$runJITlessWasm = true
$filter = nil
$envVars = []
$mode = :full
$buildType = "release"
$forceCollectContinuously = false
$reportExecutionTime = false
$ldd = nil
$artifact_exec_wrapper = nil
$numChildProcessesSetByUser = false
$maxTimeout = nil
$runUniqueId = Random.new.bytes(16).unpack("H*")[0]
$gnuParallelChunkSize = 1
$testsDebugStream = nil # set to $stderr for debugging

def putd(s)
    $testsDebugStream.puts(s) unless $testsDebugStream.nil?
end

def usage
    puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
    puts
    puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
    puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
    puts "                            --jsc specifies an already present JavaScriptCore to test."
    puts "--platform                  Indicate the target platform we are running on."
    puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
    puts "                            Skip tests tagged with //@skip if $memoryLimited"
    puts "--no-jit                    Do not run JIT specific tests."
    puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
    puts "                            platform."
    puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
    puts "--verbose            (-v)   Print more things while running."
    puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
    puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
    puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
    puts "--force-architecture        Override the architecture to run tests with."
    puts "                            e.g. x86, x86_64, arm."
    puts "--ldd                       Use alternate ldd"
    puts "--artifact-exec-wrapper     Wrapper for executing a build artifact"
    puts "--os                        Specify os instead of determining from JavaScriptCore build."
    puts "                            e.g. darwin, linux & windows."
    puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
    puts "                            In general the shell runner is slower than the make runner."
    puts "--make-runner               Uses the faster make-based runner."
    puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
    puts "--test-writer [writer]      Specifies the test script format."
    puts "                            default is to use shell scripts to run the tests"
    puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
    puts "--remote                    Specify a remote host on which to run tests from command line argument."
    puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
    puts "--report-execution-time     Print execution time for each test in ms."
    puts "--child-processes    (-c)   Specify the number of child processes."
    puts "--max-timeout               Specify a cap for the effective timeout"
    puts "--filter                    Only run tests whose name matches the given regular expression."
    puts "--help               (-h)   Print this message."
    puts "--env-vars                  Add a list of environment variables to set before running jsc."
    puts "                            Each environment variable should be separated by a space."
    puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
    puts "                            it will override the default value."
    puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
    puts "--basic                     Run with default and these additional modes: no-llint,"
    puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
    puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
    puts "                            ftl-no-cjit-small-pool."
    puts "--jitless-wasm              Run the no-wasm-jit mode"
    puts "--no-retry                  Disable auto retry"
    exit 1
end

jscArg = nil

GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
               ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
               ['--no-copy', GetoptLong::NO_ARGUMENT],
               ['--cloop', GetoptLong::NO_ARGUMENT],
               ['--platform', GetoptLong::REQUIRED_ARGUMENT],
               ['--memory-limited', GetoptLong::NO_ARGUMENT],
               ['--no-jit', GetoptLong::NO_ARGUMENT],
               ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
               ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
               ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
               ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
               ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
               ['--arch', GetoptLong::REQUIRED_ARGUMENT],
               ['--force-architecture', GetoptLong::REQUIRED_ARGUMENT],
               ['--ldd', GetoptLong::REQUIRED_ARGUMENT],
               ['--artifact-exec-wrapper', GetoptLong::REQUIRED_ARGUMENT],
               ['--os', GetoptLong::REQUIRED_ARGUMENT],
               ['--shell-runner', GetoptLong::NO_ARGUMENT],
               ['--make-runner', GetoptLong::NO_ARGUMENT],
               ['--ruby-runner', GetoptLong::NO_ARGUMENT],
               ['--gnu-parallel-runner', GetoptLong::NO_ARGUMENT],
               ['--gnu-parallel-chunk-size', GetoptLong::REQUIRED_ARGUMENT],
               ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
               ['--treat-failing-as-flaky', GetoptLong::REQUIRED_ARGUMENT],
               ['--remote', GetoptLong::REQUIRED_ARGUMENT],
               ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
               ['--report-execution-time', GetoptLong::NO_ARGUMENT],
               ['--model', GetoptLong::REQUIRED_ARGUMENT],
               ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
               ['--max-timeout', GetoptLong::REQUIRED_ARGUMENT],
               ['--filter', GetoptLong::REQUIRED_ARGUMENT],
               ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
               ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
               ['--debug', GetoptLong::NO_ARGUMENT],
               ['--release', GetoptLong::NO_ARGUMENT],
               ['--quick', '-q', GetoptLong::NO_ARGUMENT],
               ['--basic', GetoptLong::NO_ARGUMENT],
               ['--no-slow', GetoptLong::NO_ARGUMENT],
               ['--jitless-wasm', GetoptLong::NO_ARGUMENT],
               ['--no-retry', GetoptLong::NO_ARGUMENT]).each {
    | opt, arg |
    case opt
    when '--help'
        usage
    when '--jsc'
        jscArg = arg
    when '--no-copy'
        $doNotMessWithVMPath = true
    when '--output-dir'
        $outputDir = Pathname.new(arg)
    when '--cloop'
        $cloop = true
    when '--platform'
        $platform = arg
    when '--memory-limited'
        $memoryLimited = true
    when '--no-jit'
        $jitTests = false
    when '--force-collectContinuously'
        $forceCollectContinuously = true;
    when '--verbose'
        $verbosity += 1
    when '--run-bundle'
        $bundle = Pathname.new(arg)
    when '--tarball'
        $tarball = true
        $copyVM = true
        $tarFileName = arg unless arg == ''
    when '--force-vm-copy'
        $copyVM = true
    when '--shell-runner'
        $testRunnerType = :shell
    when '--make-runner'
        $testRunnerType = :make
    when '--ruby-runner'
        $testRunnerType = :ruby
    when '--gnu-parallel-runner'
        $testRunnerType = :gnuparallel
    when '--gnu-parallel-chunk-size'
        $gnuParallelChunkSize = arg.to_i
    when '--test-writer'
        $testWriter = arg
    when '--treat-failing-as-flaky'
        md = /^([^,]+),(\d+),(\d+)(,(\d+))?$/.match(arg)
        if md.nil?
            $stderr.puts("Could not parse argument to `--treat-failing-as-flaky`; expected `passPercentage,maxTries,maxFailing[,maxRetrySeconds]`")
            exit(1)
        end
        passPercentage = md[1].to_f
        if passPercentage.zero?
            $stderr.puts("Invalid passPercentage `#{md[1]}`")
            exit(1)
        end
        maxTries = md[2].to_i
        if maxTries == 0
            $stderr.puts("Invalid maxTries `#{md[2]}`")
            exit(1)
        end
        maxFailing = md[3].to_i
        if maxFailing == 0
            $stderr.puts("Invalid maxFailing `#{md[3]}`")
            exit(1)
        end
        maxRetrySeconds = nil
        if not md[5].nil?
           maxRetrySeconds = md[5].to_i
           if maxRetrySeconds == 0
               $stderr.puts("Invalid maxRetrySeconds")
               exit(1)
           end
        end
        $treatFailingAsFlaky = OpenStruct.new(:passPercentage => passPercentage,
                                              :maxTries => maxTries,
                                              :maxFailing => maxFailing,
                                              :maxRetrySeconds => maxRetrySeconds)
    when '--remote'
        $copyVM = true
        $tarball = true
        $remote = true
        uri = URI("ssh://" + arg)
        $remoteHosts << RemoteHost.new("default-#{$remoteHosts.length}", uri.user, uri.host, uri.port)
    when '--remote-config-file'
        $remoteConfigFile = arg
    when '--report-execution-time'
        $reportExecutionTime = true
    when '--child-processes'
        $numChildProcesses = arg.to_i
        $numChildProcessesSetByUser = true
    when '--max-timeout'
        $maxTimeout = arg.to_i
    when '--filter'
        $filter = Regexp.new(arg)
    when '--arch'
        $architecture = arg
    when '--force-architecture'
        $architecture = arg unless $architecture
        $forceArchitecture = arg
    when '--ldd'
        $ldd = arg
    when '--artifact-exec-wrapper'
        $artifact_exec_wrapper = arg
    when '--os'
        $hostOS = arg
    when '--model'
        $model = arg.gsub(/\A['"]+|['"]+\Z/, '')
    when '--env-vars'
        $envVars = arg.gsub(/\s+/, ' ').split(' ')
    when '--quick'
        $mode = :quick
    when '--basic'
        $mode = :basic
    when '--no-slow'
        $mode = :noSlow
    when '--jitless-wasm'
        $runJITlessWasm = true
    when '--debug'
        $buildType = "debug"
    when '--release'
        $buildType = "release"
    when '--no-retry'
        ITERATION_LIMITS.infraIterationsFloor = 1
        ITERATION_LIMITS.iterationsCeiling = 1
    end
}

if $remoteConfigFile
    file = File.read($remoteConfigFile)
    config = JSON.parse(file)

    # old style config allowing for only one remote
    if !$remote and config['remote']
        $copyVM = true
        $tarball = true
        $remote = true
        uri = URI("ssh://" + config['remote'])
        $remoteHosts = [ RemoteHost.new("default", uri.user, uri.host, uri.port) ]
        if config['remoteDirectory']
            $remoteHosts[0].remoteDirectory = config['remoteDirectory']
        end
        if config['idFilePath']
            $remoteHosts[0].identity_file_path = config['idFilePath']
        end
    end

    # we can combine --remote and a new style config
    if config['remotes']
        $copyVM = true
        $tarball = true
        $remote = true
        $remoteHosts += config['remotes'].map {
            | remote |
            uri = URI("ssh://" + remote['address'])

            host = RemoteHost.new(remote['name'], uri.user, uri.host, uri.port)
            if remote['remoteDirectory']
                host.remoteDirectory = remote['remoteDirectory']
            end
            if remote['idFilePath']
                host.identity_file_path = remote['idFilePath']
                print('Using identity file: ' + host.identity_file_path + "\r")
            end
            host
        }
    end
end

unless jscArg
    # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
    command = SCRIPTS_PATH.join("webkit-build-directory").to_s
    command += ($buildType == "release") ? " --release" : " --debug"
    command += " --executablePath"

    output = `#{command}`.split("\n")
    if !output.length
        $stderr.puts "Error: must specify --jsc <path>"
        exit 1
    end

    output = output[0]
    jscArg = Pathname.new(output).join("jsc")
    jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Helpers", "jsc") if !File.file?(jscArg)
    jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
    if !File.file?(jscArg)
        $stderr.puts "Error: must specify --jsc <path>"
        exit 1
    end

    puts "Using the following jsc path: #{jscArg}"
end

if $doNotMessWithVMPath
    $jscPath = Pathname.new(jscArg)
else
    $jscPath = Pathname.new(jscArg).realpath
end

$progressMeter = ($verbosity == 0 and $stdout.tty? and $remoteHosts.length <= 1 and !ENV["JSCTEST_noProgressMeter"])

if $bundle
    $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Helpers" + "jsc"
    $outputDir = $bundle
end

# Try to determine architecture. Return nil on failure.
def machOArchitectureCode
    begin 
        otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
        otoolLines.each_with_index {
            | value, index |
            if value =~ /magic/ and value =~ /cputype/
                return otoolLines[index + 1].split[1].to_i
            end
        }
    rescue
        $stderr.puts "Warning: unable to execute otool."
    end
    $stderr.puts "Warning: unable to determine architecture."
    nil
end

def determineArchitectureFromMachOBinary
    code = machOArchitectureCode
    return nil unless code
    is64BitFlag = 0x01000000
    case code
    when 7
        "x86"
    when 7 | is64BitFlag
        "x86_64"
    when 12
        "arm"
    when 12 | is64BitFlag
        "arm64"
    else
        $stderr.puts "Warning: unable to determine architecture from code: #{code}"
        nil
    end
end

def determineArchitectureFromELFBinary
    f = File.open($jscPath.to_s, "rb")
    data = f.read(20)

    if data[0, 4].bytes != [0x7F, 0x45, 0x4C, 0x46]
        $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    # MIPS and PowerPC may be either big- or little-endian. S390 (which includes
    # S390x) is big-endian. The rest are little-endian.
    code = data[18, 2].bytes
    case code
    when [0x03, 0]
        "x86"
    when [0x08, 0], [0, 0x08]
        "mips"
    when [0x14, 0], [0, 0x14]
        "powerpc"
    when [0x15, 0], [0, 0x15]
        "powerpc64"
    when [0, 0x16]
        "s390"
    when [0x28, 0]
        "arm"
    when [0x3E, 0]
        "x86_64"
    when [0xB7, 0]
        "arm64"
    when [0xF3, 0]
        "riscv64"
    else
        $stderr.puts "Warning: unable to determine architecture from code: #{code.inspect}"
        nil
    end
end

def determineArchitectureFromPEBinary
    f = File.open($jscPath.to_s)
    data = f.read(1024)

    if !(data[0, 2] == "MZ")
        $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian

    if !(data[peHeaderAddr, 4] == "PE\0\0")
        $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
        return nil
    end

    machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian

    case machine
    when 0x014c
        "x86"
    when 0x8664
        "x86_64"
    else
        $stderr.puts "Warning: unsupported machine type: #{machine}"
        nil
    end
end

def determineArchitecture
    case $hostOS
    when "darwin"
        determineArchitectureFromMachOBinary
    when "linux"
        determineArchitectureFromELFBinary
    when "windows"
        determineArchitectureFromPEBinary
    when "playstation"
        "x86_64"
    else
        $stderr.puts "Warning: unable to determine architecture on this platform."
        nil
    end
end

def determineOS
    case RbConfig::CONFIG["host_os"]
    when /darwin/i
        "darwin"
    when /linux/i
        "linux"
    when /mswin|mingw|cygwin/
        "windows"
    else
        $stderr.puts "Warning: unable to determine host operating system"
        nil
    end
end

if $cloop
    $jitTests = false
    $runJITlessWasm = false
    # $isWasmPlatform will already be disabled below because it is dependend on "not $cloop".
end

$hostOS = determineOS unless $hostOS
$architecture = determineArchitecture unless $architecture
$isFTLPlatform = !($architecture == "x86" || $architecture == "mips" || $architecture == "riscv64" || $hostOS == "playstation" || $architecture == "arm")
# Special case windows, we want to run the wasm tests temporarily without B3/Air support
$isWasmPlatform = (not $cloop) && $jitTests && ($isFTLPlatform || ($hostOS == "windows" && $architecture == "x86_64"))
$isOMGPlatform = $isFTLPlatform
$isSIMDPlatform = $isWasmPlatform && ($architecture == "arm64" || $architecture == "x86_64")

$skipLockdown = false
$skipMiniMode = false

if $platform == "watchos" || $architecture == "arm"
    $jitTests = false
    $isFTLPlatform = false
    $isWasmPlatform = false
    $skipLockdown = true
    $skipMiniMode = true
    $memoryLimited = true
end

# This is meant for skipping execution of tests than require a lot of address
# space. Cf. $memoryLimited, which is meant for tests that actually make use of
# a lot of memory.
# Technically, this should be the bits available in userspace, but right now we
# don't need to be this precise.
$addressBits = case $architecture
               when /.*64.*/
                   # We only have 36 bits on iOS. We'll need to differentiate
                   # between iOS and other 64-bit systems (possibly by adding an
                   # --address-bits flag to this script) when we acquire tests
                   # that require > 36 bits of address space.
                   36
               else
                   32
               end

if $architecture == "x86"
    # The JIT is temporarily disabled on this platform since
    # https://trac.webkit.org/changeset/237547
    $jitTests = false
end

def isFTLEnabled
    $jitTests && $isFTLPlatform
end

if !$testRunnerType
    if $remote and $hostOS == "darwin"
        $testRunnerType = :shell
    else
        $testRunnerType = :make
    end
end

if $remoteHosts.length > 1 and ($testRunnerType != :make) and ($testRunnerType != :gnuparallel)
    raise "Multiple remote hosts only supported with the make or gnu-parallel runners"
end

if $hostOS == "playstation" && $testWriter == "default"
    $testWriter = "playstation"
end

if $testWriter
    if /[^-a-zA-Z0-9_]/.match($testWriter)
        raise "Invalid test writer #{$testWriter} given"
    end
end

# We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
BASE_OPTIONS = ["--validateOptions=true", "--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
EAGER_OPTIONS = ["--validateOptions=true", "--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--thresholdForOMGOptimizeAfterWarmUp=20", "--thresholdForOMGOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true", "--repatchBufferingCountdown=0"]
# NOTE: Tests rely on this using scribbleFreeCells.
NO_CJIT_OPTIONS = ["--validateOptions=true", "--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
FTL_OPTIONS = ["--validateOptions=true", "--useFTLJIT=true"]
FORCE_LLINT_EXIT_OPTIONS = ["--forceOSRExitToLLInt=true"]
EXECUTABLE_FUZZER_OPTIONS = ["--useExecutableAllocationFuzz=true", "--fireExecutableAllocationFuzzRandomly=true"]

class BasePlan
    attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler, :additionalEnv, :index
    attr_accessor :retryParameters

    @@index = 0
    def initialize(directory, arguments, family, name, outputHandler, errorHandler, retryParameters, expectCrash)
        @directory = directory
        @arguments = argumentsMapper(arguments)
        @family = family
        @name = name
        @outputHandler = outputHandler
        @errorHandler = errorHandler
        @expectCrash = expectCrash
        # A plan for which @retryParameters is not nil is being
        # treated as potentially flaky.
        @retryParameters = retryParameters
        @additionalEnv = []
        @index = @@index
        @@index += 1
    end
    def self.mock(family, name, retryParameters=nil)
        self.new("/none", [], family, name, nil, nil, retryParameters, false)
    end
    def self.create(directory, arguments, family, name, outputHandler, errorHandler)
        if $runCommandOptions[:mustCrash]
            outputHandler = noisyOutputHandler
        end
        self.new(directory, arguments, family, name, outputHandler, errorHandler, $runCommandOptions[:flaky], $runCommandOptions[:mustCrash])
    end

    def expectCrash
        @expectCrash
    end
    def argumentsMapper(args)
        args
    end
    # We regularly place Plans in containers, but may modify @retryParameters
    # after the fact; only hash on @index instead.
    def hash
        @index
    end
    def to_s
        "#{@index}"
    end
end

class TestRunner
    def initialize(testRunnerType, runnerDir)
        @testRunnerType = testRunnerType
        @runnerDir = runnerDir
    end
    def prepare(runlist, serialPlans, exclusivePlans, completedPlans, remoteHosts)
        prepareScripts(runlist)
        prepareRunner(runlist, serialPlans, exclusivePlans, completedPlans, remoteHosts)
    end
    def prepareScripts(runlist)
        Dir.mkdir(@runnerDir) unless @runnerDir.directory?
        toDelete = []
        Dir.foreach(@runnerDir) {
            | filename |
            if filename =~ /^test_/
                toDelete << filename
            end
        }

        toDelete.each {
            | filename |
            File.unlink(@runnerDir + filename)
        }

        # Write test scripts in parallel as this is both an expensive and a
        # highly IO intensive operation, but each script is independent and
        # the operation is pure other than writing the unique run script.
        parallelEach(runlist) do | plan |
            plan.writeRunScript(@runnerDir + "test_script_#{plan.index}")
        end
    end
    def self.create(testRunnerType, runnerDir)
        cls = nil
        case testRunnerType
        when :shell
            cls = TestRunnerShell
        when :make
            cls = TestRunnerMake
        when :ruby
            cls = TestRunnerRuby
        when :gnuparallel
            cls = TestRunnerGnuParallel
        else
            raise "Unknown test runner type: #{testRunnerType.to_s}"
        end
        return cls.new(testRunnerType, runnerDir)
    end
end

require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"

def shouldCollectContinuously?
    $buildType == "release" or $forceCollectContinuously
end

COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false", "--verifyGC=true"] : []

$serialPlans = Set.new
$exclusivePlans = Set.new
$runlist = []

def frameworkFromJSCPath(jscPath)
    parentDirectory = jscPath.dirname
    if (parentDirectory.basename.to_s == "Resources" or parentDirectory.basename.to_s == "Helpers") and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
        parentDirectory.dirname
    elsif $hostOS == "playstation"
        jscPath.dirname
    elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
        jscPath.dirname + "JavaScriptCore.framework"
    else
        $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
        nil
    end
end


# Frequently repeated path computations. Cache results for speed.
$bundleResourcePathCache = Hash.new do |h, key|
    resourcePath, benchmarkDirectory, dir = key
    benchmarkDirectory.each_filename {
        | pathComponent |
        dir = dir.parent
    }
    h[key] = dir + resourcePath
end

def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
    $bundleResourcePathCache[[resourcePath, $benchmarkDirectory, Pathname.new(".")]]
end

def pathToVM
    pathToBundleResourceFromBenchmarkDirectory($jscPath)
end

def vmCommand
    cmd = [pathToVM.to_s]
    if not $artifact_exec_wrapper.nil?
        cmd.unshift($artifact_exec_wrapper)
    end
    if ($forceArchitecture)
        cmd = ["/usr/bin/arch", "-" + $forceArchitecture] + cmd
    end
    return cmd
end

def pathToHelpers
    pathToBundleResourceFromBenchmarkDirectory(".helpers")
end

$runCommandOptions = {}
$testSpecificRequiredOptions = []

$uniqueFilenameCounter = 0
def uniqueFilename(extension)
    payloadDir = $outputDir + "_payload"
    Dir.mkdir payloadDir unless payloadDir.directory?
    result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
    $uniqueFilenameCounter += 1
    result
end

def baseOutputName(kind)
    "#{$collectionName}/#{$benchmark}.#{kind}"
end

def addRunCommandConfig(config, *additionalEnv)
    [:kind, :command, :outputHandler, :errorHandler].each { |key|
        if not config.has_key?(key)
            raise "Missing #{key} in #{config}"
        end
    }
    kind = config[:kind]
    if $skipModes.include?(kind.to_sym) or $skipModes.include?(kind.gsub('-', '_').to_sym)
        return
    end
    $didAddRunCommand = true
    name = baseOutputName(config[:kind])
    if $filter and name !~ $filter
        return
    end
    plan = Plan.create(
        $benchmarkDirectory, config[:command], "#{$collectionName}/#{$benchmark}", name, config[:outputHandler],
        config[:errorHandler])
    if config.has_key?(:additionalEnv)
        plan.additionalEnv.push(*(config[:additionalEnv]))
    end
    if $runCommandOptions[:serial]
        # Add this to the list of tests to be run on their own, so
        # that we can treat them specially when scheduling, but keep
        # it in the $runlist for code that dosn't care about
        # scheduling.
        $serialPlans.add(plan)
    elsif $runCommandOptions[:exclusive]
        $exclusivePlans.add(plan)
    end

    if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
        $runlist.unshift plan
    else
        $runlist << plan
    end
end

def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
    config = {
        :kind => kind,
        :command => command,
        :outputHandler => outputHandler,
        :errorHandler => errorHandler,
        :additionalEnv => additionalEnv,
    }
    addRunCommandConfig(config)
end

# Returns true if there were run commands found in the file ($benchmarkDirectory +
# $benchmark), in which case those run commands have already been executed. Otherwise
# returns false, in which case you're supposed to add your own run commands.
def parseRunCommands
    oldDidAddRunCommand = $didAddRunCommand
    $didAddRunCommand = false
    $skipped = false

    Dir.chdir($outputDir) {
        File.open($benchmarkDirectory + $benchmark) {
            | inp |
            inp.each_line {
                | line |
                begin
                    doesMatch = line =~ /^\/\/@/
                rescue Exception => e
                    # Apparently this happens in the case of some UTF8 stuff in some files, where
                    # Ruby tries to be strict and throw exceptions.
                    next
                end
                next unless doesMatch
                eval $~.post_match
                if $skipped
                    break
                end
            }
        }
    }

    result = $didAddRunCommand
    $didAddRunCommand = result or oldDidAddRunCommand
    result
end

def slow!
    $runCommandOptions[:isSlow] = true
    skip() if ($mode == :quick or $mode == :basic or $mode == :noSlow)
end

def mustCrash!
    $testSpecificRequiredOptions += ["--signal-expected"]
    $runCommandOptions[:mustCrash] = true
end

def serial!
    $runCommandOptions[:serial] = true
end

def exclusive!
    $runCommandOptions[:exclusive] = true
end

def memoryHog!
    exclusive!
    if $memoryLimited
        skip
    end
end


# Retry parameters for tests that we treat as flaky, either because
# they've been explicitly marked so (via flaky!) or because we were
# asked to --treat-failing-as-flaky.
class RetryParameters
    attr_reader :passPercentage, :maxTries
    def initialize(passPercentage, maxTries)
        @passPercentage = passPercentage
        @maxTries = maxTries
    end
    def to_s
        "RetryParameters(#{@passPercentage}, #{@maxTries})"
    end
    def result(statuses)
        if statuses.length > @maxTries
            # This could happen if (because of remotes disappearing
            # and reappearing) there are multiple runs for the last
            # try for a flaky test. Just ignore any extra statuses.
            statuses = statuses.take(@maxTries)
        end
        successes = successfulStatuses(statuses)
        remaining = @maxTries - statuses.length
        requiredSuccesses = (@passPercentage * @maxTries).ceil
        ret = nil # "has not completed yet"
        if successes >= requiredSuccesses
            ret = true
        elsif (successes + remaining) < requiredSuccesses
            ret = false
        end
        putd("#{self}.result(#{statuses}) => #{ret}")
        ret
    end
    private
    def successfulStatuses(statuses)
        statuses.count {
            |status|
            status == STATUS_FILE_PASS
        }
    end
end

def flaky!(passPercentage, maxTries)
    $runCommandOptions[:flaky] = RetryParameters.new(passPercentage, maxTries)
end

def requireOptions(*options)
    $testSpecificRequiredOptions += options
end

def runWithOptions(config, *options)
    baseOptions = BASE_OPTIONS
    if config.has_key?(:no_base_options)
        baseOptions = []
    end
    commandPrefix = config.fetch(:command_prefix, [])
    if config.has_key?(:place_benchmark_early)
        config[:command] = commandPrefix + vmCommand + [$benchmark.to_s] + baseOptions + options + $testSpecificRequiredOptions
    else
        config[:command] = commandPrefix + vmCommand + baseOptions + options + $testSpecificRequiredOptions + [$benchmark.to_s]
    end
    addRunCommandConfig(config)
end

def runWithOutputHandler(kind, outputHandler, *options)
    config = {
        :kind => kind,
        :outputHandler => outputHandler,
        :errorHandler => simpleErrorHandler,
    }
    runWithOptions(config, *options)
end

def runWithOutputHandlerWithoutBaseOption(kind, outputHandler, *options)
    config = {
        :kind => kind,
        :outputHandler => outputHandler,
        :errorHandler => simpleErrorHandler,
        :no_base_options => true,
    }
    runWithOptions(config, *options)
end

def run(kind, *options)
    runWithOutputHandler(kind, silentOutputHandler, *options)
end

def runInner(config, *options)
    config = config.dup
    if not config.has_key?(:outputHandler)
        config[:outputHandler] = silentOutputHandler
    end
    if not config.has_key?(:errorHandler)
        config[:errorHandler] = simpleErrorHandler
    end
    runWithOptions(config, *options)
end

def runWithoutBaseOptionConfig(config, *options)
    config = config.dup
    config[:no_base_options] = true
    runWithOptions(config, *options)
end

def runWithoutBaseOption(kind, *options)
    runWithOutputHandlerWithoutBaseOption(kind, silentOutputHandler, *options)
end

def runOneLargeHeap(*optionalTestSpecificOptions)
    if $memoryLimited
        $didAddRunCommand = true
        puts "Skipping #{$collectionName}/#{$benchmark}"
    else
        run("default", *optionalTestSpecificOptions)
    end
end

def bytecodeCacheTemplate
    if ($hostOS == "darwin")
        return "bytecode-cache"
    elsif ($hostOS == "linux")
        return "bytecode-cacheXXXXXX"
    end
    nil
end

def runBytecodeCacheImpl(optionalTestSpecificOptions, *additionalEnv)
    fileTemplate = bytecodeCacheTemplate
    if fileTemplate.nil?
        return nil
    end
    {
        :config => {
            :command_prefix => [
                "sh",
                (pathToHelpers + "bytecode-cache-test-helper.sh").to_s,
                fileTemplate.to_s,
            ],
            :place_benchmark_early => true,
            :additionalEnv => additionalEnv,
        },
        :testSpecificOptions => FTL_OPTIONS + optionalTestSpecificOptions,
    }
end


def configInitializerPlain
    Proc.new { |config, kind|
        { :kind => kind}
    }
end

def configInitializerConfig
    Proc.new { |config, kind|
        config = config.dup
        config[:kind] = kind
        config
    }
end

# For each base mode (defined below) we generate two kinds of functions:
#
# - a version which takes a config argument and passes it along, only
#   setting the kind field
# - a "plain" version which starts out with an empty config
#
# The plain  version is intended  for use in the  testcase definitions
# (in `//@` comments and the like).
#
# The former version is used for plumbing. The caller may set various
# fields in the config which will be respected.
#
# This way, we can
# - define a set of test modes in defaultRunConfig
# - have defaultRunConfig propagate the config argument to the run*Config
#   functions it calls
# - call defaultRunConfig from e.g. defaultRunNoisyTest with the output
#   handlers appropriately set, in order to make sure we're running
#   the exact same of tests.
ConfigKind = Struct.new(:extension, :expectConfig, :initializer)
configKinds = [
    ConfigKind.new("", false, configInitializerPlain),
    ConfigKind.new("Config", true, configInitializerConfig),
]

# Define base test modes. Each mode is an array of [name, kind,
# options]. The name is used to derive the ruby method names, the kind
# is used for reporting (i.e. what you'd see in this script's
# output). In the common case, options is a static array; if not, it's
# a Proc that returns a dict that needs to be unpacked (see its use
# site for a more detailed description).
BASE_MODES = [
    [
        "NoCJIT",
        "ftl-no-cjit",
        [
            "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--libpasForcePGMWithRate=#{10 + rand(10)}",

        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS
    ],
    [
        "FTLNoCJIT",
        "misc-ftl-no-cjit",
        [
            "--useHandlerICInFTL=true",
            "--allowNonSPTagging=false",
            "--libpasForcePGMWithRate=#{10 + rand(10)}",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITValidate",
        "ftl-no-cjit-validate-sampling-profiler",
        [
            "--validateGraph=true",
            "--validateBCE=true",
            "--useSamplingProfiler=true",
            "--useHandlerICInFTL=true",
            "--forceUnlinkedDFG=true",
            "--allowNonSPTagging=false",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITNoPutStackValidate",
        "ftl-no-cjit-no-put-stack-validate",
        [
            "--validateGraph=true",
            "--usePutStackSinking=false",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITNoInlineValidate",
        "ftl-no-cjit-no-inline-validate",
        [
            "--validateGraph=true",
            "--maximumInliningDepth=1",
            "--useB3HoistLoopInvariantValues=true",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITOSRValidation",
        "ftl-no-cjit-osr-validation",
        [
            "--validateFTLOSRExitLiveness=true",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "DFGEager",
        "dfg-eager",
        EAGER_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS +
        FORCE_LLINT_EXIT_OPTIONS
    ],
    [
        "DFGEagerNoCJITValidate",
        "dfg-eager-no-cjit-validate",
        [
            "--validateGraph=true",
        ] +
        NO_CJIT_OPTIONS +
        EAGER_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS
    ],
    [
        "FTLEager",
        "ftl-eager",
        [
            "--useRandomizingExecutableIslandAllocation=true",
            "--forcePolyProto=true",
            "--useHandlerICInFTL=true",
            "--allowNonSPTagging=false",
        ] +
        FTL_OPTIONS +
        EAGER_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS
    ],
    [
        "FTLEagerNoCJITValidate",
        "ftl-eager-no-cjit",
        [
            "--validateGraph=true",
            "--validateBCE=true",
            "--libpasForcePGMWithRate=#{10 + rand(10)}",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS +
        EAGER_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS +
        FORCE_LLINT_EXIT_OPTIONS +
        EXECUTABLE_FUZZER_OPTIONS
    ],
    [
        "FTLEagerNoCJITOSRValidation",
        "ftl-eager-no-cjit-osr-validation",
        [
            "--validateFTLOSRExitLiveness=true",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS +
        EAGER_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS
    ],
    [
        "NoCJITNoASO",
        "no-cjit-no-aso",
        [
            "--useArchitectureSpecificOptimizations=false",
        ] +
        NO_CJIT_OPTIONS
    ],
    [
        "NoCJITNoAccessInlining",
        "no-cjit-no-access-inlining",
        [
            "--useAccessInlining=false",
        ] +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITNoAccessInlining",
        "ftl-no-cjit-no-access-inlining",
        [
            "--useAccessInlining=false",
            "--allowNonSPTagging=false",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "FTLNoCJITSmallPool",
        "ftl-no-cjit-small-pool",
        [
            $buildType == "debug" ? "--jitMemoryReservationSize=4194304" : "--jitMemoryReservationSize=1048576",
            "--maxDFGNodesInBasicBlockForPreciseAnalysis=100",
            "--useArrayAllocationProfiling=false",
        ] +
        FTL_OPTIONS +
        NO_CJIT_OPTIONS
    ],
    [
        "NoCJIT",
        "no-cjit",
        NO_CJIT_OPTIONS
    ],
    [
        "EagerJettisonNoCJIT",
        "eager-jettison-no-cjit",
        [
            "--useRandomizingExecutableIslandAllocation=true",
            "--forceCodeBlockToJettisonDueToOldAge=true",
            "--verifyGC=true",
        ] +
        NO_CJIT_OPTIONS
    ],
    [
        "ShadowChicken",
        "shadow-chicken",
        [
            "--useDFGJIT=false",
            "--alwaysUseShadowChicken=true",
        ]
    ],
    [
        "Lockdown",
        "lockdown",
        [
            "--useJIT=false",
            "--useGenerationalGC=false",
            "--useConcurrentGC=false",
            "--useLLIntICs=false",
            "--useZombieMode=true",
            "--allowDoubleShape=false",
            "--alwaysHaveABadTime=true",
            "--libpasForcePGMWithRate=#{10 + rand(10)}",
        ]
    ],
    [
        "MiniMode",
        "mini-mode",
        [
            "--forceMiniVMMode=true",
        ]
    ],
    [
        "LogicalAssignmentOperatorsEnabled",
        "logical-assignment-operators-enabled",
        [
            "--useLogicalAssignmentOperators=true",
        ] +
        FTL_OPTIONS
    ],
    [
        "NoJIT",
        "no-jit",
        [
            "--useJIT=false",
        ]
    ],
    [
        # NOTE: Tests rely on this using scribbleFreeCells.
        "NoCJITValidate",
        "no-cjit",
        [
            "--validateBytecode=true",
            "--validateGraph=true",
        ] +
        NO_CJIT_OPTIONS
    ],
    [
        "NoCJITValidatePhases",
        "no-cjit-validate-phases",
        [
            "--validateBytecode=true",
            "--validateGraphAtEachPhase=true",
            "--useSourceProviderCache=false",
            "--useRandomizingExecutableIslandAllocation=true",
            "--useLLIntICs=false",
        ] +
        NO_CJIT_OPTIONS
    ],
    [
        "NoCJITCollectContinuously",
        "no-cjit-collect-continuously",
        NO_CJIT_OPTIONS +
        COLLECT_CONTINUOUSLY_OPTIONS
    ],
    [
        "Default",
        "default",
        FTL_OPTIONS
    ],
    [
        "NoFTL",
        "no-ftl",
        []
    ],
    [
        "WithRAMSize",
        nil, # Not used
        Proc.new { |size, *optionalTestSpecificOptions|
            {
                :config => {
                    :kind => "ram-size-#{size}",
                },
                :testSpecificOptions => [
                    "--forceRAMSize=#{size}",
                ] + optionalTestSpecificOptions
            }
        }
    ],
    [
        "BytecodeCache",
        "bytecode-cache",
        Proc.new { |*optionalTestSpecificOptions|
            runBytecodeCacheImpl(optionalTestSpecificOptions)
        }
    ],
    [
        "BytecodeCacheNoAssertion",
        "bytecode-cache",
        Proc.new { |*optionalTestSpecificOptions|
            runBytecodeCacheImpl(optionalTestSpecificOptions, "JSC_forceDiskCache=false")
        }
    ],
    [
        "FTLEagerWatchdog",
        nil,
        Proc.new { |*optionalTestSpecificOptions|
            timeout = rand(100)
            {
                :config => {
                    :kind => "ftl-eager-watchdog-#{timeout}",
                },
                :testSpecificOptions => [
                    "--watchdog=#{timeout}",
                    "--watchdog-exception-ok",
                ] +
                FTL_OPTIONS +
                EAGER_OPTIONS +
                COLLECT_CONTINUOUSLY_OPTIONS +
                optionalTestSpecificOptions
            }
        }
    ],
    [
        "NoLLInt",
        "no-llint",
        Proc.new { |*optionalTestSpecificOptions|
            if $jitTests
                {
                    :config => {
                    },
                    :testSpecificOptions => [
                        "--useLLInt=false",
                    ] + optionalTestSpecificOptions
                }
            else
                nil
            end
        }
    ],
    [
        "OneLangeHeap",
        "default",
        Proc.new { |*optionalTestSpecificOptions|
            if $memoryLimited
                nil
            else
                {
                    :testSpecificOptions => optionalTestSpecificOptions
                }
            end
        }
    ]
]

BASE_MODES.each { |mode|
    name = "run#{mode[0]}"
    kind = mode[1]
    options = mode[2]

    # We need to define two variants, one expecting a config as the first
    # argument, one not.
    configKinds.each { |configKind|
        methodName = "#{name}#{configKind.extension}".to_sym
        define_method(methodName) { |*args|
            config = nil
            if configKind.expectConfig
                # If we're defining a method that expects a config
                # argument, pick it out of the args to pass to the
                # initializer.
                config = args.shift
            end
            # The config is initialized differently depending on whether
            # we're in a run*Config method or not.
            config = configKind.initializer.call(config, kind)
            finalOptions = nil
            if options.respond_to?(:call)
                dynamicOptions = options.call(*args)
                if dynamicOptions.nil?
                    skip
                    return
                end
                # The Proc object may override any config option passed
                # in. This is used e.g. for dynamic test names as used
                # by WithRAMSize and FTLEagerWatchdog.
                config.merge!(dynamicOptions[:config])
                # As the Proc may consume arguments, it's responsible
                # for returning the final option list. Needed e.g. by
                # WithRAMSize.
                finalOptions = dynamicOptions[:testSpecificOptions]
            else
                finalOptions = options + args
            end
            runInner(config, *finalOptions)
        }
    }
}

CFG_NOISY = {
            :outputHandler => noisyOutputHandler,
            :errorHandler => noisyErrorHandler,
}.freeze

BASE_MODES.each { |mode|
    name = "runNoisyTest#{mode[0]}".to_sym
    define_method(name) { |*args|
        # For each base mode, define the "noisy" variant which simply
        # calls the respective run#{name}Config, passing in the "noisy"
        # config.
        send("run#{mode[0]}Config", CFG_NOISY, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *args)
    }
}

# Default set of tests to run; propagates the config to every callee.
def defaultRunConfig(config, subsetOptions, *optionalTestSpecificOptions)
    config.freeze
    if not subsetOptions.has_key?(:ignoreQuickMode) and $mode == :quick
        defaultQuickRunConfig(config, *optionalTestSpecificOptions)
    else
        runDefaultConfig(config, *optionalTestSpecificOptions)
        runBytecodeCacheConfig(config, *optionalTestSpecificOptions)
        runMiniModeConfig(config, *optionalTestSpecificOptions) unless $skipMiniMode
        runLockdownConfig(config, *optionalTestSpecificOptions) unless $skipLockdown
        if $jitTests
            if not subsetOptions.has_key?(:skipNoLLInt)
                runNoLLIntConfig(config, *optionalTestSpecificOptions)
            end
            runNoCJITValidatePhasesConfig(config, *optionalTestSpecificOptions)
            runNoCJITCollectContinuouslyConfig(config, *optionalTestSpecificOptions) if shouldCollectContinuously?
            if not subsetOptions.has_key?(:skipEager)
                runDFGEagerConfig(config)
                if $mode != :basic
                    runDFGEagerNoCJITValidateConfig(config, *optionalTestSpecificOptions)
                    runEagerJettisonNoCJITConfig(config, *optionalTestSpecificOptions)
                end
            end

            return if !$isFTLPlatform

            runNoFTLConfig(config, *optionalTestSpecificOptions)
            if not subsetOptions.has_key?(:skipEager)
                runFTLEagerConfig(config, *optionalTestSpecificOptions)
                runFTLEagerNoCJITValidateConfig(config, *optionalTestSpecificOptions) if $buildType == "release"
            end
            runFTLNoCJITSmallPoolConfig(config, *optionalTestSpecificOptions)

            return if $mode == :basic

            runFTLNoCJITValidateConfig(config, *optionalTestSpecificOptions)
            runFTLNoCJITNoPutStackValidateConfig(config, *optionalTestSpecificOptions)
            runFTLNoCJITNoInlineValidateConfig(config, *optionalTestSpecificOptions)
        end
    end
end

def defaultRun
    defaultRunConfig({}, {})
end

def defaultNoNoLLIntRun
    defaultRunConfig({}, {:skipNoLLInt => true})
end

def defaultQuickRunConfig(config, *optionalTestSpecificOptions)
    runDefaultConfig(config, *optionalTestSpecificOptions)
    if $jitTests
        runNoCJITValidateConfig(config, *optionalTestSpecificOptions)

        return if !$isFTLPlatform

        runNoFTLConfig(config, *optionalTestSpecificOptions)
        runFTLNoCJITValidateConfig(config, *optionalTestSpecificOptions)
    end
end

def defaultQuickRun
    defaultQuickRunConfig({})
end

def defaultSpotCheckNoMaximalFlush
    defaultQuickRun

    return if !$jitTests
    runNoCJITNoAccessInlining

    return if !$isFTLPlatform
    runFTLNoCJITOSRValidation
    runFTLNoCJITNoAccessInlining
end

def defaultSpotCheck
    defaultSpotCheckNoMaximalFlush
    runEagerJettisonNoCJIT if $jitTests
end

# This is expected to not do eager runs because eager runs can have a lot of recompilations
# for reasons that don't arise in the real world. It's used for tests that assert convergence
# by counting recompilations.
def defaultNoEagerRun(*optionalTestSpecificOptions)
    defaultRunConfig({}, { :skipEager => true }, *optionalTestSpecificOptions)
end

def defaultNoSamplingProfilerRun(*optionalTestSpecificOptions)
    defaultRunConfig({}, { :ignoreQuickMode => true }, *optionalTestSpecificOptions)
end

def runProfiler
    if $remote or $memoryLimited or ($hostOS == "windows") or ($hostOS == "playstation")
        skip
        return
    end

    profilerOutput = uniqueFilename(".json")
    if $canRunDisplayProfilerOutput
        addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, *vmCommand, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], noisyOutputHandler, simpleErrorHandler)
    else
        puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
        run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
    end
end

def runExceptionFuzz
    subCommand = escapeAll(vmCommand + ["--validateOptions=true", "--useDollarVM=true", "--useExceptionFuzz=true", $benchmark.to_s])
    addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end

def runExecutableAllocationFuzz(name, *options)
    subCommand = escapeAll(vmCommand + ["--validateOptions=true", "--useDollarVM=true", $benchmark.to_s] + options)
    addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
end

def runTypeProfiler
    if !$jitTests
        return
    end

    run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
    run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))

    return if !$isFTLPlatform

    run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
end

def runControlFlowProfiler
    if !$jitTests
        return
    end

    return if !$isFTLPlatform

    run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def runTest262(mode, exception, includeFiles, flags)
    failsWithException = exception != "NoException"
    isStrict = false
    isModule = false
    isAsync = false

    flags.each {
        | flag |
        case flag
        when :strict
            isStrict = true
        when :module
            isModule = true
        when :async
            isAsync = true
        else
            raise "Invalid flag for runTest262, #{flag}"
        end
    }

    prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)

    args = vmCommand + BASE_OPTIONS
    args << "--exception=" + exception if failsWithException
    args << "--test262-async" if isAsync
    args += $testSpecificRequiredOptions
    args += includeFiles

    case mode
    when :normal
        errorHandler = simpleErrorHandler
        outputHandler = silentOutputHandler
    when :fail
        errorHandler = expectedFailErrorHandler
        outputHandler = noisyOutputHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = expectedFailErrorHandler
        outputHandler = noisyOutputHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end

    if isStrict
        kind = "default-strict"
        args << "--strict-file=#{$benchmark}"
    else
        kind = "default"
        if isModule
            args << "--module-file=#{$benchmark}"
        else
            args << $benchmark.to_s
        end
    end

    addRunCommand(kind, args, outputHandler, errorHandler)
end

def prepareTest262Fixture
    # This function is used to add the files used by Test262 modules tests.
    prepareExtraRelativeFiles([""], $collection)
end

def runES6(mode)
    args = vmCommand + BASE_OPTIONS + $testSpecificRequiredOptions + [$benchmark.to_s]
    case mode
    when :normal
        errorHandler = simpleErrorHandler
    when :fail
        errorHandler = expectedFailErrorHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = expectedFailErrorHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end
    addRunCommand("default", args, noisyOutputHandler, errorHandler)
end

def defaultRunModules(noLLInt: true)
    run("default-modules", "-m")

    if !$jitTests
        return
    end

    run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
    run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
    run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
    run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))

    return if !$isFTLPlatform

    run("default-ftl-modules", "-m", *FTL_OPTIONS)
    run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
    run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
    run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
    run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
    run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=1048576", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def noNoLLIntRunModules
    defaultRunModules(noLLInt: false)
end

def runWebAssembly
    return unless $isWasmPlatform
    run("default-wasm", "-m", *FTL_OPTIONS)
    if $mode != :quick
        run("wasm-no-cjit", "-m", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager", "-m", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + EAGER_OPTIONS))
        run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *FTL_OPTIONS)
        run("wasm-slow-memory", "-m", "--useWasmFastMemory=false", *FTL_OPTIONS)
        run("wasm-collect-continuously", "-m", "--collectContinuously=true", "--zeroExecutableMemoryOnFree=true", "--verifyGC=true", *FTL_OPTIONS) if shouldCollectContinuously?
        run("wasm-bbq", "-m", "--useWasmIPInt=false", *FTL_OPTIONS)
        if $isOMGPlatform
            run("wasm-omg", "-m", "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *FTL_OPTIONS)
        end
        if $isFTLPlatform
            run("wasm-simd", "-m", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *FTL_OPTIONS) unless !$isSIMDPlatform
            run("wasm-aggressive-inline", "-m", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS))
        end
    end
end

def runWebAssemblyJetStream2
    return unless $isWasmPlatform

    if $memoryLimited
        skip
        return
    end

    prepareExtraAbsoluteFiles(JETSTREAM2_PATH, ["JetStreamDriver.js"])
    prepareExtraRelativeFilesWithBaseDirectory(Dir[JETSTREAM2_PATH + "wasm" + "*.js"].map { |f| "wasm/" + File.basename(f) }, $collection.dirname, $extraFilesBaseDir.dirname)
    prepareExtraRelativeFilesWithBaseDirectory(Dir[JETSTREAM2_PATH + "wasm" + "*.wasm"].map { |f| "wasm/" + File.basename(f) }, $collection.dirname, $extraFilesBaseDir.dirname)

    run("default-wasm", *FTL_OPTIONS)

    if $mode != :quick
        run("wasm-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager", "--useRandomizingExecutableIslandAllocation=true", *(FTL_OPTIONS + EAGER_OPTIONS))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--verifyGC=true", *FTL_OPTIONS)
        run("wasm-slow-memory", "--useWasmFastMemory=false", *FTL_OPTIONS)
        run("wasm-collect-continuously", "--collectContinuously=true", "--verifyGC=true", *FTL_OPTIONS) if shouldCollectContinuously?
        run("wasm-bbq", "--useWasmIPInt=false", *FTL_OPTIONS)
        if $isOMGPlatform
            run("wasm-omg", "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *FTL_OPTIONS)
        end
        if $isFTLPlatform
            run("wasm-simd", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *FTL_OPTIONS) unless !$isSIMDPlatform
            run("wasm-aggressive-inline", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS))
        end
    end
end

def runDefaultWasm(*optionalTestSpecificOptions)
    return unless $isWasmPlatform
    run("default-wasm", *(FTL_OPTIONS + optionalTestSpecificOptions))
end

def runWebAssemblySuite(*optionalTestSpecificOptions)
    return unless $isWasmPlatform
    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    modules << "wasm.json"
    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
    if optionalTestSpecificOptions[0] == :no_module
      optionalTestSpecificOptions.shift
    else
      optionalTestSpecificOptions.unshift "-m"
    end
    runDefaultWasm(*optionalTestSpecificOptions)
    if $mode != :quick
        run("wasm-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-slow-memory", "--useWasmFastMemory=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-collect-continuously", "--collectContinuously=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
        run("wasm-bbq", "--useWasmIPInt=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-bbq-no-consts", "--useWasmIPInt=false", "--disableBBQConsts=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        if $runJITlessWasm
          run("wasm-no-jit", "--useJIT=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
          run("wasm-no-wasm-jit", "--useBBQJIT=false", "--useOMGJIT=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
        if $isOMGPlatform
            run("wasm-omg", "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
        if $isFTLPlatform
            run("wasm-simd", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *(FTL_OPTIONS + optionalTestSpecificOptions)) unless !$isSIMDPlatform
            run("wasm-aggressive-inline", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        end
    end
end

def runV8WebAssemblySuite(*optionalTestSpecificOptions)
    return unless $isWasmPlatform
    prepareExtraAbsoluteFiles(Pathname.new(WASMTESTS_PATH).join("v8", "resources"), ["async-compile.js", "mjsunit.js", "trap-location.js", "user-properties-common.js", "wasm-module-builder.js"])
    if optionalTestSpecificOptions[0] == :no_module
      optionalTestSpecificOptions.shift
    else
      optionalTestSpecificOptions.unshift "-m"
    end
    runDefaultWasm(*optionalTestSpecificOptions)
    if $mode != :quick
        run("wasm-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-slow-memory", "--useWasmFastMemory=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-collect-continuously", "--collectContinuously=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
        run("wasm-bbq", "--useWasmIPInt=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        run("wasm-bbq-no-consts", "--useWasmIPInt=false", "--disableBBQConsts=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        if $runJITlessWasm
          run("wasm-no-jit", "--useJIT=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
          run("wasm-no-wasm-jit", "--useBBQJIT=false", "--useOMGJIT=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
        if $isOMGPlatform
            run("wasm-omg", "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end        
        if $isFTLPlatform
            run("wasm-simd", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *(FTL_OPTIONS + optionalTestSpecificOptions)) unless !$isSIMDPlatform
            run("wasm-aggressive-inline", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        end
    end
end

def runV8WebAssemblySuiteQuick(*optionalTestSpecificOptions)
    return unless $isWasmPlatform
    prepareExtraAbsoluteFiles(Pathname.new(WASMTESTS_PATH).join("v8", "resources"), ["async-compile.js", "mjsunit.js", "trap-location.js", "user-properties-common.js", "wasm-module-builder.js"])
    if optionalTestSpecificOptions[0] == :no_module
      optionalTestSpecificOptions.shift
    else
      optionalTestSpecificOptions.unshift "-m"
    end
    runDefaultWasm(*optionalTestSpecificOptions)
end

def runWasmHarnessTest(kind, *options)
    raise unless $isWasmPlatform
    if $skipModes.include?(kind.to_sym) or $skipModes.include?(kind.gsub('-', '_').to_sym)
        return
    end

    wasmFiles = allWasmFiles($collection)
    wasmFiles.each {
        | file |
        basename = file.basename.to_s
        addRunCommand("(" + basename + ")-" + kind, vmCommand + options + $testSpecificRequiredOptions + [$benchmark.to_s, "--", basename], silentOutputHandler, simpleErrorHandler)
    }
end

def runWebAssemblyWithHarness(*optionalTestSpecificOptions)
    raise unless $benchmark.to_s =~ /harness\.m?js/
    return unless $isWasmPlatform

    wasmFiles = allWasmFiles($collection)
    prepareExtraRelativeFiles(wasmFiles.map { |f| f.basename }, $collection)

    runWasmHarnessTest("default-wasm", *(FTL_OPTIONS + optionalTestSpecificOptions))
    if $mode != :quick
        runWasmHarnessTest("wasm-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        runWasmHarnessTest("wasm-eager", *(FTL_OPTIONS + EAGER_OPTIONS + optionalTestSpecificOptions))
        runWasmHarnessTest("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runWasmHarnessTest("wasm-slow-memory", "--useWasmFastMemory=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runWasmHarnessTest("wasm-collect-continuously", "--collectContinuously=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
        runWasmHarnessTest("wasm-bbq", "--useWasmIPInt=false", *(FTL_OPTIONS + optionalTestSpecificOptions)) # Doesn't actually depend on FTL, but we enable it on the same platforms for now.
        if $isOMGPlatform
            runWasmHarnessTest("wasm-omg", "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
        if $isFTLPlatform
            runWasmHarnessTest("wasm-simd", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *FTL_OPTIONS) unless !$isSIMDPlatform
            runWasmHarnessTest("wasm-aggressive-inline", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS))
        end
    end
end

def runWebAssemblyEmscripten(mode)
    case mode
    when :skip
        return
    end
    return unless $isWasmPlatform
    wasm = $benchmark.to_s.sub! '.js', '.wasm'
    prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
    run("default-wasm", *FTL_OPTIONS)
    if $mode != :quick
        run("wasm-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *FTL_OPTIONS)
        run("wasm-collect-continuously", "--collectContinuously=true", "--verifyGC=true", *FTL_OPTIONS) if shouldCollectContinuously?
        run("wasm-bbq", "--useWasmIPInt=false", *FTL_OPTIONS) # Doesn't actually depend on FTL, but we enable it on the same platforms for now.
        if $runJITlessWasm
          run("wasm-no-jit", "--useJIT=false" *FTL_OPTIONS)
          run("wasm-no-wasm-jit", "--useBBQJIT=false", "--useOMGJIT=false" *FTL_OPTIONS)
        end
        if $isOMGPlatform
            run("wasm-omg", "--useWasmIPInt=true", "--useBBQJIT=false", *FTL_OPTIONS)
        end
        if $isFTLPlatform
            run("wasm-simd", "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *FTL_OPTIONS) unless !$isSIMDPlatform
            run("wasm-aggressive-inline", "--wasmInliningMaximumWasmCalleeSize=2147483647", "--wasmInliningBudget=100000", "--wasmInliningMaximumDepth=10", *(FTL_OPTIONS + EAGER_OPTIONS))
        end
    end
end

def runWebAssemblySpecTestBase(mode, specHarnessPath, postfix, *optionalTestSpecificOptions)
    case mode
    when :skip
        return
    end
    return unless $isWasmPlatform

    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    modules << "wasm.json"
    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)

    harness = Dir[WASMTESTS_PATH + (specHarnessPath + "/") + "*.js"].map { |f| File.basename(f) }
    prepareExtraRelativeFiles(harness.map { |f| ("../" + specHarnessPath + "/") + f }, $collection)

    specHarnessJsPath = "../" + specHarnessPath + ".js"
    runWithOutputHandler("default-wasm" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, *(FTL_OPTIONS + optionalTestSpecificOptions))
    if $mode != :quick
        runWithOutputHandler("wasm-no-cjit" + (postfix || ''), noisyOutputHandler, specHarnessJsPath,  *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
        # FIXME: We should probably only run these two for GC spec tests.
        runWithOutputHandler("wasm-eager-jettison" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runWithOutputHandler("wasm-collect-continuously" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--collectContinuously=true", "--verifyGC=true", *(FTL_OPTIONS + optionalTestSpecificOptions)) if shouldCollectContinuously?
        runWithOutputHandler("wasm-bbq" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--useWasmIPInt=false", *(FTL_OPTIONS + optionalTestSpecificOptions))
        runWithOutputHandler("wasm-bbq-no-consts" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--useWasmIPInt=false", "--disableBBQConsts=true", *(FTL_OPTIONS + optionalTestSpecificOptions))
        if $isOMGPlatform
            runWithOutputHandler("wasm-omg" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
        if $isFTLPlatform
            runWithOutputHandler("wasm-simd" + (postfix || ''), noisyOutputHandler, specHarnessJsPath, "--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1", *(FTL_OPTIONS + optionalTestSpecificOptions))
        end
    end
end

def runWebAssemblyThreadsSpecTest(mode)
    runWebAssemblySpecTestBase(mode, "threads-spec-harness", nil)
end

def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
    return unless $isWasmPlatform
    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
    modules << "wasm.json"
    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
    # Only let WebAssembly get executable memory.
    run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=80000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
end

# Support for running the imported WPT WebAssembly tests
# (LayoutTests/imported/w3c/web-platform-tests/wasm) in the jsc shell. Results are diffed against
# the committed -expected.txt baselines.
WPT_PATH = LAYOUTTESTS_PATH + "imported/w3c/web-platform-tests"

# Origin the WPT server serves the imported tree from during a browser layout-test run, as
# configured by LayoutTests/imported/w3c/resources/config.json. Stack locations that the WAST
# harness bakes into assertion descriptions are these URLs in the committed baselines, so
# wpt-harness-post.js rewrites the shell's relative load paths against this.
WPT_BASE_URL = "http://web-platform.test:8800/"

# The wasm JIT-configuration matrix, mirroring runWebAssemblySpecTestBase. Returns [kind, options].
def wasmWPTConfigs(*extraOptions)
    configs = [["default-wasm", FTL_OPTIONS + extraOptions]]
    if $mode != :quick
        configs << ["wasm-no-cjit", FTL_OPTIONS + NO_CJIT_OPTIONS + extraOptions]
        configs << ["wasm-eager-jettison", ["--forceCodeBlockToJettisonDueToOldAge=true", "--useRandomizingExecutableIslandAllocation=true", "--verifyGC=true"] + FTL_OPTIONS + extraOptions]
        configs << ["wasm-collect-continuously", ["--collectContinuously=true", "--verifyGC=true"] + FTL_OPTIONS + extraOptions] if shouldCollectContinuously?
        configs << ["wasm-bbq", ["--useWasmIPInt=false"] + FTL_OPTIONS + extraOptions]
        configs << ["wasm-bbq-no-consts", ["--useWasmIPInt=false", "--disableBBQConsts=true"] + FTL_OPTIONS + extraOptions]
        configs << ["wasm-omg", ["--useWasmIPInt=true", "--useBBQJIT=true", "--thresholdForBBQOptimizeAfterWarmUp=0", "--thresholdForBBQOptimizeSoon=0", "--thresholdForOMGOptimizeAfterWarmUp=0", "--thresholdForOMGOptimizeSoon=0"] + FTL_OPTIONS + extraOptions] if $isOMGPlatform
        configs << ["wasm-simd", ["--useWasmSIMD=1", "--forceAllFunctionsToUseSIMD=1"] + FTL_OPTIONS + extraOptions] if $isFTLPlatform
    end
    configs
end

# Leading `// META: ...` comment lines of a WPT test (parsing stops at the first non-comment line).
def wptMetaLines(path)
    lines = []
    File.open(path) { | inp |
        inp.each_line { | line |
            break unless line =~ /^\s*\/\//
            lines << line
        }
    }
    lines
end

def wptIsJSShell?(path)
    line = wptMetaLines(path).find { | l | l =~ /\/\/\s*META:\s*global=/ }
    !line.nil? && line.include?("jsshell")
end

def wptMetaScripts(path)
    wptMetaLines(path).map { | l | l =~ /\/\/\s*META:\s*script=(\S+)/ ? $1 : nil }.compact
end

# Title WPT gives the generated page: the test's file name (basename, no subdirectory) without the
# .any/.window/.worker.js suffix. The shell names an unnamed block-body test "Untitled";
# wpt-harness-post.js remaps that to this.
def wptTitle(benchmark)
    File.basename(benchmark.to_s).sub(/\.js$/, '').sub(/\.(any|window|worker)$/, '')
end

# Copies a file (given as a path relative to sourceDir) into the collection root in the bundle
# ($benchmarkDirectory), so it can be referenced by that same relative path from the run's working
# directory. Recursive collections keep $benchmarkDirectory at the collection root and carry the
# subdirectory in $benchmark, so harness files copied here are shared by every test in the tree.
def wptCopyBeside(relativeName, sourceDir)
    prepareExtraRelativeFilesWithBaseDirectory([relativeName], $benchmarkDirectory, sourceDir)
end

# The collection root's path within the served WPT tree (e.g. "wasm/core/js"). Load paths in the
# run directory are relative to this root, so it is also the prefix that turns one into the URL
# the WPT server would serve it from.
def wptCollectionPrefix
    $extraFilesBaseDir.relative_path_from(WPT_PATH.realpath)
end

# Maps each `// META: script=` dependency to a load path relative to the run's working directory
# (the collection root). prepareCollection copies the whole collection tree into the bundle, so a
# dependency within it is already present: a /-absolute (/wasm/...) dep resolves to its
# collection-relative path, a relative dep to the benchmark's own directory. A /-absolute dep
# pointing outside the collection would not be in the bundle, so raise rather than reference a
# missing file (no jsshell test does this today).
def wptPrepareScripts(scripts)
    scripts.map { | s |
        if s.start_with?("/")
            rel = Pathname.new(s.sub(%r{^/}, '')).relative_path_from(wptCollectionPrefix)
            raise "WPT script dependency #{s} is outside the #{wptCollectionPrefix} collection" if rel.to_s.start_with?("..")
            rel.to_s
        else
            ($benchmark.dirname + s).to_s
        end
    }
end

def runWebAssemblyWPT(expectedFile, harnessBeforeBenchmark, *extraOptions)
    FileUtils.mkdir_p((Pathname.new($collectionName) + $benchmark.dirname).to_s)

    # Copy the harness into the collection root so these references resolve from the run directory.
    # Use the imported WPT copy of testharness.js (not LayoutTests/resources/testharness.js): it is
    # the version the browser loads for these imported tests and thus the one that generated the
    # committed baselines.
    wptCopyBeside("resources/testharness.js", WPT_PATH)
    wptCopyBeside("wpt-harness-pre.js", WASMTESTS_PATH + "wpt")
    wptCopyBeside("wpt-harness-mid.js", WASMTESTS_PATH + "wpt")
    wptCopyBeside("wpt-harness-post.js", WASMTESTS_PATH + "wpt")

    before = ["wpt-harness-pre.js", "resources/testharness.js"] + harnessBeforeBenchmark + ["wpt-harness-mid.js"]
    after = ["wpt-harness-post.js", "--", wptTitle($benchmark), WPT_BASE_URL + wptCollectionPrefix.to_s + "/"]

    # Enable the in-development JSC features the same way WebKitTestRunner does for the browser
    # layout-test run, so these tests exercise the same feature set (and match the baselines the
    # browser generated) rather than only the shipped defaults.
    wasmWPTConfigs(*extraOptions).each { | kind, options |
        args = vmCommand + BASE_OPTIONS + ["--enable-all-experimental-features"] + options + $testSpecificRequiredOptions +
            before + [$benchmark.to_s] + after
        addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(expectedFile))
    }
end

def runWebAssemblyWPTJSAPI(*extraOptions)
    return unless $isWasmPlatform
    raise unless $benchmark.to_s =~ /\.js$/
    testName = $~.pre_match
    return unless wptIsJSShell?($benchmarkDirectory + $benchmark)
    return if $benchmark.to_s =~ /\.tentative\./

    # The -expected.txt baseline sits next to the test and is already in the bundle via cp_r.
    # jsshell-only tests (global=jsshell, no window/worker) are never run in the browser and so
    # have no committed baseline to diff against; skip them.
    expectedFile = ($benchmarkDirectory + "#{testName}-expected.txt").to_s
    return unless File.exist?(expectedFile)

    scripts = wptPrepareScripts(wptMetaScripts($benchmarkDirectory + $benchmark))
    runWebAssemblyWPT(expectedFile, scripts, *extraOptions)
end

def runWebAssemblyWPTCore(*extraOptions)
    return unless $isWasmPlatform
    raise unless $benchmark.to_s =~ /\.js$/

    # Core baselines live in the parallel core/<proposal>/ tree: the test's path with the "js/"
    # segment dropped (core/js[/<proposal>]/foo.wast.js -> core[/<proposal>]/foo.wast.js-expected.txt).
    # $benchmark carries the proposal subdirectory, and $extraFilesBaseDir is the collection source
    # (.../core/js) whose dirname (.../core) is that parallel tree's root; joining them yields the
    # baseline. Copy it into the bundle at the same relative path so the diff path matches.
    baselineSource = $extraFilesBaseDir.dirname + "#{$benchmark}-expected.txt"
    return unless File.exist?(baselineSource.to_s)
    wptCopyBeside("#{$benchmark}-expected.txt", $extraFilesBaseDir.dirname)
    expectedFile = ($benchmarkDirectory + "#{$benchmark}-expected.txt").to_s

    # harness/async_index.js is inside the collection, so prepareCollection already copied it into
    # the bundle; reference it directly.
    runWebAssemblyWPT(expectedFile, ["harness/async_index.js"], *extraOptions)
end

def runChakra(mode, exception, baselineFile, extraFiles)
    raise unless $benchmark.to_s =~ /\.js$/
    failsWithException = exception != "NoException"
    testName = $~.pre_match

    prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
    prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)

    args = vmCommand + BASE_OPTIONS
    args += FTL_OPTIONS if $isFTLPlatform
    args += EAGER_OPTIONS
    args << "--exception=" + exception if failsWithException
    args << "--dumpException" if failsWithException
    args += $testSpecificRequiredOptions
    args += ["jsc-lib.js"]

    case mode
    when :baseline
        prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
        errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
        outputHandler = noisyOutputHandler
    when :pass
        errorHandler = chakraPassFailErrorHandler
        outputHandler = noisyOutputHandler
    when :skipDueToOutdatedOrBadTest
        return
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end

    kind = "default"
    args << $benchmark.to_s

    addRunCommand(kind, args, outputHandler, errorHandler)
end

def runLayoutTest(kind, *options)
    raise unless $benchmark.to_s =~ /\.js$/
    testName = $~.pre_match
    if kind
        kind = "layout-" + kind
    else
        kind = "layout"
    end

    prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
    prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])

    args = vmCommand + BASE_OPTIONS + options + $testSpecificRequiredOptions +
        [(Pathname.new("resources") + "standalone-pre.js").to_s,
         $benchmark.to_s,
         (Pathname.new("resources") + "standalone-post.js").to_s]
    addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
end

def runLayoutTestNoFTL
    runLayoutTest("no-ftl")
end

def runLayoutTestNoLLInt
    runLayoutTest("no-llint", "--useLLInt=false")
end

def runLayoutTestNoCJIT
    runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
end

def runLayoutTestDFGEagerNoCJIT
    runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def runLayoutTestDefault
    runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
end

def runLayoutTestFTLNoCJIT
    runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
end

def runLayoutTestFTLEagerNoCJIT
    runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def noFTLRunLayoutTest
    if !$jitTests
        return
    end

    runLayoutTestNoLLInt
    runLayoutTestNoCJIT
    runLayoutTestDFGEagerNoCJIT
end

def defaultQuickRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        if $isFTLPlatform
            runLayoutTestNoFTL
            runLayoutTestFTLNoCJIT
            runLayoutTestFTLEagerNoCJIT
        else
            noFTLRunLayoutTest
        end
    end
end

def defaultRunLayoutTest
    if $mode == :quick
        defaultQuickRunLayoutTest
    else
        runLayoutTestDefault
        if $jitTests
            noFTLRunLayoutTest

            return if !$isFTLPlatform

            runLayoutTestNoFTL
            runLayoutTestFTLNoCJIT
            runLayoutTestFTLEagerNoCJIT
        end
    end
end

def noEagerNoNoLLIntTestsRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        runLayoutTestNoCJIT

        return if !$isFTLPlatform

        runLayoutTestNoFTL
        runLayoutTestFTLNoCJIT
    end
end

def noNoLLIntRunLayoutTest
    runLayoutTestDefault
    if $jitTests
        runLayoutTestNoCJIT
        runLayoutTestDFGEagerNoCJIT

        return if !$isFTLPlatform

        runLayoutTestNoFTL
        runLayoutTestFTLNoCJIT
        runLayoutTestFTLEagerNoCJIT
    end
end

def prepareExtraRelativeFilesWithBaseDirectory(extraFiles, destination, baseDirectory)
    Dir.chdir($outputDir) {
        extraFiles.each {
            | file |
            dest = destination + file
            FileUtils.mkdir_p(dest.dirname)
            FileUtils.cp baseDirectory + file, dest
        }
    }
end

def prepareExtraRelativeFiles(extraFiles, destination)
    prepareExtraRelativeFilesWithBaseDirectory(extraFiles, destination, $extraFilesBaseDir)
end

def baseDirForCollection(collectionName)
    Pathname(".tests") + collectionName
end

def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
    raise unless absoluteBase.absolute?
    Dir.chdir($outputDir) {
        collectionBaseDir = baseDirForCollection($collectionName)
        extraFiles.each {
            | file |
            destination = collectionBaseDir + file
            FileUtils.mkdir_p destination.dirname unless destination.directory?
            FileUtils.cp absoluteBase + file, destination
        }
    }
end

def runComplexTest(before, after, additionalEnv, *options)
    prepareExtraRelativeFiles(before.map{|v| (Pathname("..") + v).to_s}, $collection)
    prepareExtraRelativeFiles(after.map{|v| (Pathname("..") + v).to_s}, $collection)
    args = vmCommand + BASE_OPTIONS + options + $testSpecificRequiredOptions + before.map{|v| v.to_s} + [$benchmark.to_s] + after.map{|v| v.to_s}
    addRunCommand("complex", args, noisyOutputHandler, simpleErrorHandler, *additionalEnv)
end

def runMozillaTest(kind, mode, extraFiles, *options)
    if kind
        kind = "mozilla-" + kind
    else
        kind = "mozilla"
    end
    prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
    args = vmCommand + BASE_OPTIONS + options + $testSpecificRequiredOptions + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
    case mode
    when :normal
        errorHandler = mozillaErrorHandler
    when :negative
        errorHandler = mozillaExit3ErrorHandler
    when :fail
        errorHandler = mozillaFailErrorHandler
    when :failDueToOutdatedOrBadTest
        errorHandler = mozillaFailErrorHandler
    when :skip
        return
    else
        raise "Invalid mode: #{mode}"
    end
    addRunCommand(kind, args, noisyOutputHandler, errorHandler)
end

def runMozillaTestDefault(mode, *extraFiles)
    runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
end

def runMozillaTestNoFTL(mode, *extraFiles)
    runMozillaTest("no-ftl", mode, extraFiles)
end

def runMozillaTestLLInt(mode, *extraFiles)
    runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
end

def runMozillaTestBaselineJIT(mode, *extraFiles)
    runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
end

def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
    runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
    runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
end

def defaultQuickRunMozillaTest(mode, *extraFiles)
    if $jitTests
        runMozillaTestDefault(mode, *extraFiles)
        runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
    else
        runMozillaTestNoFTL(mode, *extraFiles)
        if $jitTests
            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
        end
    end
end

def defaultRunMozillaTest(mode, *extraFiles)
    if $mode == :quick
        defaultQuickRunMozillaTest(mode, *extraFiles)
    else
        runMozillaTestNoFTL(mode, *extraFiles)
        if $jitTests
            runMozillaTestLLInt(mode, *extraFiles)
            runMozillaTestBaselineJIT(mode, *extraFiles)
            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
            runMozillaTestDefault(mode, *extraFiles)
            runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
        end
    end
end

def runNoisyTestWithEnv(kind, *additionalEnv)
    config = CFG_NOISY.dup
    config[:kind] = kind
    config[:additionalEnv] = additionalEnv
    runDefaultConfig(config)
end

def defaultRunNoisyTest(*optionalTestSpecificOptions)
    config = {
        :outputHandler => noisyOutputHandler,
        :errorHandler => noisyErrorHandler,
    }
    defaultRunConfig(config, {}, *optionalTestSpecificOptions)
end

def skip
    $didAddRunCommand = true
    $skipped = true
    puts "Skipping #{$collectionName}/#{$benchmark}"
end

def allWasmFiles(path)
    if path.file?
        [path]
    else
        result = []
        Dir.foreach(path) {
            | filename |
            next unless filename =~ /\.m?wasm$/
            next unless (path + filename).file?
            result << path + filename
        }
        result
    end
end

def allJSFiles(path)
    if path.file?
        [path]
    else
        result = []
        Dir.foreach(path) {
            | filename |
            next unless filename =~ /\.m?js$/
            next unless (path + filename).file?
            result << path + filename
        }
        result
    end
end

# Like allJSFiles, but descends into subdirectories. Used by collections whose yaml entry sets
# "recursive: true" (e.g. the imported WPT wasm tests, whose proposal subdirectories would
# otherwise each need their own entry). Sorted for deterministic test ordering.
def allJSFilesRecursive(path)
    if path.file?
        [path]
    else
        Dir.glob((path + "**" + "*.{js,mjs}").to_s).sort.map { | f | Pathname.new(f) }.select { | f | f.file? }
    end
end

def uniqueifyName(names, name)
    result = name.to_s
    toAdd = 1
    while names[result]
        result = "#{name}-#{toAdd}"
        toAdd += 1
    end
    names[result] = true
    result
end

def simplifyCollectionName(collectionPath)
    outerDir = collectionPath.dirname
    name = collectionPath.basename
    lastName = name
    if collectionPath.directory?
        while lastName.to_s =~ /test/
            lastName = outerDir.basename
            name = lastName + name
            outerDir = outerDir.dirname
        end
    end
    uniqueifyName($collectionNames, name)
end

def prepareCollection(name)
    FileUtils.mkdir_p $outputDir + name

    absoluteCollection = $collection.realpath

    Dir.chdir($outputDir) {
        bundleDir = baseDirForCollection(name)

        # Create the proper directory structures.
        FileUtils.mkdir_p bundleDir
        if bundleDir.basename == $collection.basename
            FileUtils.cp_r absoluteCollection, bundleDir.dirname
            $collection = bundleDir
        else
            FileUtils.cp_r absoluteCollection, bundleDir
            $collection = bundleDir + $collection.basename
        end

        $extraFilesBaseDir = absoluteCollection
    }
end

$collectionNames = {}

def isWasmTest(path)
    pathStr = path.to_s;
    wasmIndex = pathStr.index("wasm");
    if not wasmIndex
        return false
    end
    pathStr[wasmIndex-1,1] == "\-" or pathStr[wasmIndex+4,1] == "\-"
end

def handleCollectionFile(collection)
    collectionName = simplifyCollectionName(collection)
   
    paths = {}
    subCollections = []
    YAML::load(IO::read(collection)).each {
        | entry |
        if entry["collection"]
            subCollections << entry["collection"]
            next
        end
        
        if Pathname.new(entry["path"]).absolute?
            raise "Absolute path: " + entry["path"] + " in #{collection}"
        end
        
        if paths[entry["path"]]
            raise "Duplicate path: " + entry["path"] + " in #{collection}"
        end
        
        subCollection = collection.dirname + entry["path"]
        
        if subCollection.file?
            subCollectionName = Pathname.new(entry["path"]).dirname
        else
            subCollectionName = entry["path"]
        end
        
        $collection = subCollection
        $collectionName = Pathname.new(collectionName)
        Pathname.new(subCollectionName).each_filename {
            | filename |
            next if filename =~ /^\./
            $collectionName += filename
        }
        $collectionName = $collectionName.to_s
        
        prepareCollection($collectionName)
      
        Dir.chdir($outputDir) {
            pathsToSearch = [$collection]
            if entry["tests"]
                if entry["tests"].is_a? Array
                    pathsToSearch = entry["tests"].map {
                        | testName |
                        pathsToSearch[0] + testName
                    }
                else
                    pathsToSearch[0] += entry["tests"]
                end
            end
            pathsToSearch.each {
                | pathToSearch |
                jsFiles = entry["recursive"] ? allJSFilesRecursive(pathToSearch) : allJSFiles(pathToSearch)
                jsFiles.each {
                    | path |

                    next if not $isWasmPlatform and isWasmTest(path)
                    if entry["recursive"]
                        # Preserve the subdirectory path in the benchmark identity so nested tests
                        # sharing a basename (e.g. core/js/memory_grow and core/js/multi-memory/
                        # memory_grow) do not collide in their output files, and the bundle layout
                        # mirrors the source tree. $benchmarkDirectory stays at the collection root.
                        $benchmark = path.relative_path_from(pathToSearch)
                        $benchmarkDirectory = pathToSearch
                    else
                        $benchmark = path.basename
                        $benchmarkDirectory = path.dirname
                    end
                    $skipModes = Set.new
                    $runCommandOptions = {}
                    $testSpecificRequiredOptions = []
                    eval entry["cmd"]
                }
            }
        }
    }
    
    subCollections.each {
        | subCollection |
        handleCollection(collection.dirname + subCollection)
    }
end

def handleCollectionDirectory(collection)
    collectionName = simplifyCollectionName(collection)
    
    $collection = collection
    $collectionName = collectionName
    prepareCollection(collectionName)
   
    Dir.chdir($outputDir) {
        $benchmarkDirectory = $collection
        allJSFiles($collection).each {
            | path |

            next if not $isWasmPlatform and isWasmTest(path)
            $benchmark = path.basename
            $skipModes = Set.new
            $runCommandOptions = {}
            $testSpecificRequiredOptions = []
            defaultRun unless parseRunCommands
        }
    }
end

def handleCollection(collection)
    collection = Pathname.new(collection)
    
    if collection.file?
        handleCollectionFile(collection)
    else
        handleCollectionDirectory(collection)
    end
end

def prepareBundle
    raise if $bundle

    if $doNotMessWithVMPath
        if !$remote and !$tarball
            $testingFrameworkPath = (frameworkFromJSCPath($jscPath) || $jscPath.dirname).realpath
            $jscPath = Pathname.new($jscPath).realpath
        else
            $testingFrameworkPath = frameworkFromJSCPath($jscPath)
        end
    else
        originalJSCPath = $jscPath
        vmDir = $outputDir + ".vm"
        FileUtils.mkdir_p vmDir
        
        frameworkPath = frameworkFromJSCPath($jscPath)
        destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
        # On Apple Cocoa CMake builds the jsc binary lives next to the
        # framework rather than inside <framework>/Helpers/jsc as on Xcode
        # builds. Detect that and stage jsc at .vm/jsc so we don't have to
        # mutate the framework that .vm/JavaScriptCore.framework symlinks to.
        jscOutsideFramework = frameworkPath && !File.file?(frameworkPath + "Helpers" + "jsc")
        if jscOutsideFramework
            $jscPath = Pathname.new(".vm") + "jsc"
        else
            $jscPath = destinationFrameworkPath + "Helpers" + "jsc"
        end
        $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath

        if frameworkPath
            source = frameworkPath
            destination = Pathname.new(".vm")
        elsif $hostOS == "windows"
            # Make sure to copy dll along with jsc on Windows
            originalJSCDir = File.dirname(originalJSCPath)
            source = [originalJSCPath] + [originalJSCDir + "/jscLib.dll"]

            # Check for and copy JavaScriptCore.dll and WTF.dll for dynamic builds
            javaScriptCoreDLLPath = File.join(originalJSCDir, "JavaScriptCore.dll")
            wtfDLLPath = File.join(originalJSCDir, "WTF.dll")
            if (File.exist?(javaScriptCoreDLLPath))
                source = source + [javaScriptCoreDLLPath]
            end
            if (File.exist?(wtfDLLPath))
                source = source + [wtfDLLPath]
            end

            destination = $jscPath.dirname

            Dir.chdir($outputDir) {
                FileUtils.mkdir_p destination
            }
        else
            source = originalJSCPath
            destination = $jscPath

            Dir.chdir($outputDir) {
                FileUtils.mkdir_p $jscPath.dirname
            }
        end

        Dir.chdir($outputDir) {
            if $copyVM
                FileUtils.cp_r source, destination
            else
                begin
                    FileUtils.ln_s source, destination
                rescue Exception
                    $stderr.puts "Warning: unable to create soft link, trying to copy."
                    FileUtils.cp_r source, destination
                end
            end

            if jscOutsideFramework
                jscDestination = Pathname.new(".vm") + "jsc"
                if $copyVM
                    FileUtils.cp originalJSCPath, jscDestination
                else
                    begin
                        FileUtils.ln_s originalJSCPath, jscDestination
                    rescue Exception
                        $stderr.puts "Warning: unable to create soft link for jsc, trying to copy."
                        FileUtils.cp originalJSCPath, jscDestination
                    end
                end
            end

            if $hostOS == "linux"
                if $remote
                    bundle_binary = (Pathname.new(THIS_SCRIPT_PATH).dirname + 'bundle-binary').realpath
                    Dir.mktmpdir {
                        | tmpdir |
                        # Generate bundle in a temporary directory so that
                        # we can safely pick it up regardless of its name
                        # (it's the only zip file there).
                        cmdline = [
                            bundle_binary.to_s,
                            "--dest-dir=#{$jscPath.dirname}",
                            "--log-level=debug",
                            $jscPath.to_s
                        ]
                        if not $ldd.nil?
                            cmdline << "--ldd=#{$ldd}"
                        end
                        mysys(cmdline)
                    }
                else
                    if originalJSCPath.dirname.basename.to_s == "bin" && originalJSCPath.basename.to_s == "jsc"
                        libPath = originalJSCPath.dirname.dirname + "lib"
                        if libPath.directory?
                            ENV["LD_LIBRARY_PATH"] ||= "#{libPath}"
                        end
                    end
                end
            end
        }
    end
    
    Dir.chdir($outputDir) {
        FileUtils.cp_r HELPERS_PATH, ".helpers"
    }

    ARGV.each {
        | collection |
        handleCollection(collection)
    }

    if $runlist.size == 0
        $stderr.puts("No tests selected for execution (overzealous --filter?)")
        exit(2)
    end

    puts
end

def cleanOldResults
    raise unless $bundle

    eachResultFile($outputDir) {
        | path |
        FileUtils.rm_f path
    }
end

def cleanEmptyResultFiles
    eachResultFile($outputDir) {
        | path |
        next unless path.basename.to_s =~ /\.out$/
        next unless FileTest.size(path) == 0
        FileUtils.rm_f path
    }
end

def eachResultFile(startingDir, &block)
    dirsToClean = [startingDir]
    until dirsToClean.empty?
        nextDir = dirsToClean.pop
        Dir.foreach(nextDir) {
            | entry |
            next if entry =~ /^\./
            path = nextDir + entry
            if path.directory?
                dirsToClean.push(path)
            else
                block.call(path)
            end
        }
    end
end

def parallelEach(array, &block)
    # Short of using 'require "parallel"' we use a simple statically
    # partitioned multiprocess dispatch for processing fixed chunks of the
    # given  array in parallel. We use Process rather than Thread to
    # parallelise CPU load as well as IO (due to the GIL).

    # Some platforms (notably Windows) do not support Process.fork, so work
    # serially on these.
    nWorkers = Process.respond_to?(:fork) ? $numChildProcesses : 1

    # Chunk size is rounded up
    chunkSize = (array.size + (nWorkers - 1)) / nWorkers

    # If chunk size is too small, work serially
    if chunkSize <= 1
      nWorkers = 1
      chunkSize = array.size
    end

    childPIDs = []

    # Chunks 1 to nWorkers-1 run in the worker processes
    for i in 1...nWorkers do
      chunkStart = i*chunkSize
      break if chunkStart >= array.size
      pid = Process.fork
      if pid.nil?
        # Worker process. Process chunk i.
        array.slice(chunkStart, chunkSize).each(&block)
        Process.exit!(true)
      else
        childPIDs << pid
      end
    end

    # Main process. Process chunk 0.
    array.slice(0, chunkSize).each(&block)

    # Wait for workers
    for pid in childPIDs do
      _, status = Process.waitpid2(pid)
      raise "Child process still running" unless status.exited?
      if status.exitstatus != 0
        STDERR.puts "Child process failed with status: #{status.exitstatus}"
        exit(status.exitstatus)
      end
    end
end

def cleanRunnerDirectory
    raise unless $bundle
    Dir.foreach($runnerDir) {
        | filename |
        next unless filename =~ /^#{STATUS_FILE}/
        FileUtils.rm_f $runnerDir + filename
    }
end

def sshRead(cmd, remoteHost, options={})
    raise unless $remote

    result = ""
    ssh_cmd = ["ssh"] +
              SSH_OPTIONS_DEFAULT +
              ["-p", remoteHost.port.to_s] +
              (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) +
             ["#{remoteHost.user}@#{remoteHost.host}", cmd]
    IO.popen(ssh_cmd, "r") { |inp|
        result = inp.read
    }
    raise "#{$?}" unless $?.success? or options[:ignoreFailure]
    result
end

def runCommandOnTester(cmd)
    if $remote
        $remoteHosts.each { |remoteHost|
            begin
                # Return first successful value. Obviously, this
                # assumes the remotes are homogeneous.
                return sshRead(cmd, remoteHost)
            rescue Exception => e
                $stderr.puts("Error running `#{cmd}` on #{remoteHost.host}: #{e}")
            end
        }
        return "0"
    end
    `#{cmd}`
end

def commandToGetNumberOfProcessors
    if $hostOS == "windows"
        "cmd /c echo %NUMBER_OF_PROCESSORS%"
    else
        "sysctl -n hw.activecpu 2>/dev/null || nproc --all 2>/dev/null || getconf _NPROCESSORS_ONLN"
    end
end

def numberOfProcessors
    begin
        numProcessors = runCommandOnTester(commandToGetNumberOfProcessors).to_i
    rescue
        numProcessors = 0
    end

    if numProcessors == 0
        $stderr.puts("Warning: could not determine the number of remote CPUs, defaulting to 1")
        numProcessors = 1
    end
    return numProcessors
end

def runAndMonitorCommandOutput(cmd, &blk)
    cmd = cmd.collect { |a|
        a.to_s
    }
    IO.popen(cmd, "r") {
        | p |
        p.each_line {
            | line |
            blk.call(p.pid, line)
            puts(line)
            $stdout.flush
        }
    }
end

$runnerDirMutex = Mutex.new
def runAndMonitorTestRunnerCommand(cmd, options={})
    numberOfTests = 0
    $runnerDirMutex.synchronize {
        Dir.chdir($runnerDir) {
            # -1 for the runscript, and -2 for '..' and '.'
            numberOfTests = Dir.entries(".").count - 3
        }
    }
    unless $progressMeter
        mysys(cmd.join(' '), options)
    else
       running = {}
       didRun = {}
       didFail = {}
       blankLine = true
       prevStringLength = 0
       printAboveProgressMeter = Proc.new {
           | line |
           unless blankLine
               print("\r" + " " * prevStringLength + "\r")
           end
           puts line
           blankLine = true
       }
       IO.popen(cmd.join(' '), mode="r") {
           | inp |
           inp.each_line {
               | line |
               line = line.scrub.chomp
               if line =~ /^Running /
                   running[$~.post_match] = true
               elsif line =~ /^PASS: /
                   didRun[$~.post_match] = true
                   printAboveProgressMeter.call(line) if $reportExecutionTime
               elsif line =~ /^FAIL: /
                   didRun[$~.post_match] = true
                   didFail[$~.post_match] = true
               else
                   printAboveProgressMeter.call(line)
               end

               def lpad(str, chars)
                   str = str.to_s
                   if str.length > chars
                       str
                   else
                      "%#{chars}s"%(str)
                   end
               end

               string  = ""
               string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
               unless didFail.empty?
                   string += " (failed #{didFail.size})"
               end
               string += " "
               (running.size - didRun.size).times {
                   string += "."
               }
               if string.length < prevStringLength
                   print string
                   print(" " * (prevStringLength - string.length))
               end
               print string
               prevStringLength = string.length
               blankLine = false
               $stdout.flush
           }
       }
       puts
       if not $?.success? and not options[:ignoreFailure]
           raise "Failed to run #{cmd}: #{$?.inspect}"
       end
    end
end

def getRemoteDirectoryIfNeeded(remoteHost)
    if !remoteHost.remoteDirectory
        remoteHost.remoteDirectory = JSON::parse(sshRead("cat ~/.bencher", remoteHost))["tempPath"]
    end
end

def copyBundleToRemote(remoteHost)
    mysys(["ssh"] + SSH_OPTIONS_DEFAULT + (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) + ["-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}"])
    mysys(["scp"] + SSH_OPTIONS_DEFAULT + (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) + ["-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}"])
end

def getLocalRemoteResultPath(remoteHost)
    $outputDir.dirname + "#{remoteHost.host}-#{remoteHost.port}"
end

def copyRemoteResultToHost(remoteHost)
    hostResultFolder = getLocalRemoteResultPath(remoteHost)
    Dir.mkdir hostResultFolder unless hostResultFolder.directory?
    mysys(["scp"] + SSH_OPTIONS_DEFAULT + (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) + ["-P", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}/#{$outputDir.basename}/.runner/#{STATUS_FILE}", "#{getLocalRemoteResultPath(remoteHost).to_s}/#{STATUS_FILE}" ])
end

def clearLocalRemoteResults(remoteHost)
    hostResultFolder = getLocalRemoteResultPath(remoteHost)
    FileUtils.rm_rf hostResultFolder
end

def exportBaseEnvironmentVariables(escape)
    if escape
        dyldFrameworkPath = "\\$(cd #{$testingFrameworkPath.dirname}; pwd)"
        ldLibraryPath = "\\$(cd #{$testingFrameworkPath.dirname}/..; pwd)/#{$jscPath.dirname}"
    else
        dyldFrameworkPath = "\$(cd #{$testingFrameworkPath.dirname}; pwd)"
        ldLibraryPath = "\$(cd #{$testingFrameworkPath.dirname}/..; pwd)/#{$jscPath.dirname}"
    end
    [
        "export DYLD_FRAMEWORK_PATH=#{dyldFrameworkPath} && ",
        "export LD_LIBRARY_PATH=#{ldLibraryPath} &&",
        "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && ",
        "export JSCTEST_hardTimeout=#{Shellwords.shellescape(ENV['JSCTEST_hardTimeout'])} && ",
        "export JSCTEST_memoryLimit=#{Shellwords.shellescape(ENV['JSCTEST_memoryLimit'])} && ",
        "export JSCTEST_CrashReportArgV=#{Shellwords.shellescape(ENV['JSCTEST_CrashReportArgV'])} && ",
        "export TZ=#{Shellwords.shellescape(ENV['TZ'])} && ",
    ].join("")
end

def runTestRunner(testRunner, remoteHosts, remoteIndex=0)
    if not remoteHosts.nil?
        remoteHost = remoteHosts[remoteIndex]
        getRemoteDirectoryIfNeeded(remoteHost)
        remoteScript = "\""
        remoteScript += "cd #{remoteHost.remoteDirectory}/#{$outputDir.basename}/.runner && "
        remoteScript += exportBaseEnvironmentVariables(true)
        $envVars.each { |var| remoteScript += "export " << var << "\n" }
        remoteScript += "#{testRunner.command(remoteIndex)}\""
        runAndMonitorTestRunnerCommand(["ssh"] + SSH_OPTIONS_DEFAULT + (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) + ["-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript])
    else
        Dir.chdir($runnerDir) {
            runAndMonitorTestRunnerCommand(Shellwords.shellsplit(testRunner.command))
        }
    end
end

STATUS_RE = /^(?<index>\d+)\s(?<runId>\h+)\s(?<exitCode>\d+)\s(?<result>#{STATUS_FILE_PASS}|#{STATUS_FILE_FAIL})$/

def processStatusLine(map, line)
    md = STATUS_RE.match(line)
    if md.nil?
        # Malfromed lines can legitimately occur when the remote fails at an
        # inopportune moment. Ignore those lines but print them out, so that we
        # can easily diagnose actual issues with the remote output.
        $stderr.puts("Ignoring malformed status line `#{line}`")
        return
    end
    index = md[:index].to_i
    runId = md[:runId]
    result = md[:result]
    if runId != $runUniqueId
        if $bundle
            # The scripts in the bundle have their own runId embedded,
            # just ignore the value.
        else
            # This may conceivably happen if a remote goes
            # away in the middle of a run and comes back
            # online in the middle of a different run.
            $stderr.puts("Ignoring stale status file for #{index} (ID #{runId} but current ID is #{$runUniqueId})")
            return
        end
    end
    if map.has_key?(index)
        map[index].push(result)
    else
        map[index] = [result]
    end
end

def getStatusMap(map={})
    if $remote
        # Note: here we're using $remoteHosts (instead of getting the
        # list of live remoteHosts from the caller, because there may
        # well be test results on a remoteHost that got rebooted
        # (note, the test results are tagged with a run ID, so we'll
        # ignore any stale results from a previous run).
        forEachRemote($remoteHosts, :dropOnFailure => true, :timeout => 8 * REMOTE_TIMEOUT) { |_, host|
            copyRemoteResultToHost(host)
            File.open(getLocalRemoteResultPath(host) + STATUS_FILE) do |f|
                f.each do |line|
                    processStatusLine(map, line.strip)
                end
            end
            clearLocalRemoteResults(host)
        }
    else
        Dir.chdir($runnerDir) {
            File.open(STATUS_FILE) do |f|
                f.each do |line|
                    processStatusLine(map, line.strip)
                end
            end
        }
    end
    map
end

def detectFailures(statusMap={})
    raise if $bundle
    if statusMap.size == 0
        statusMap = getStatusMap
    end

    evaluator = TestResultEvaluatorFinal.new($runlist, statusMap)
    evaluator.visit!({:treatIncompleteAsFailed => true})
    evaluator.validate

    if evaluator.noresult > 0
        $stderr.puts("Could not get the exit status for #{evaluator.noresult} tests")
        # We can't change our exit code, as run-javascriptcore-tests
        # expects 0 even when there are failures.
    end

    familyMap = evaluator.familyMap

    File.open($outputDir + "resultsByFamily", "w") {
        | outp |
        first = true
        familyMap.keys.sort.each {
            | familyName |
            if first
                first = false
            else
                outp.puts
            end

            outp.print "#{familyName}:"

            numPassed = 0
            familyMap[familyName].each {
                | entry |
                if entry[:result] == "PASS"
                    numPassed += 1
                end
            }

            if numPassed == familyMap[familyName].size
                outp.puts " PASSED"
            elsif numPassed == 0
                outp.puts " FAILED"
            else
                outp.puts
                familyMap[familyName].each {
                    | entry |
                    outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
                }
            end
        }
    }
end

def compressBundle
    cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
    $stderr.puts ">> #{cmd}" if $verbosity >= 2
    raise unless system(cmd)
end

def clean(file)
    FileUtils.rm_rf file unless $bundle
end

clean($outputDir + "failed")
clean($outputDir + "passed")
clean($outputDir + "noresult")
clean($outputDir + "flaky")
clean($outputDir + "results")
clean($outputDir + "resultsByFamily")
clean($outputDir + ".vm")
clean($outputDir + ".helpers")
clean($outputDir + ".runner")
clean($outputDir + ".tests")
clean($outputDir + "_payload")

Dir.mkdir($outputDir) unless $outputDir.directory?

$outputDir = $outputDir.realpath
$runnerDir = $outputDir + ".runner"

if !$numChildProcesses
    if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
        $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
        $numChildProcessesSetByUser = true
    elsif ENV["NUMBER_OF_PROCESSORS"]
        $numChildProcesses = ENV["NUMBER_OF_PROCESSORS"].to_i
        $numChildProcessesSetByUser = true
    else
        $numChildProcesses = numberOfProcessors
    end
end

if ENV["JSCTEST_timeout"]
    # In the worst case, the processors just interfere with each other.
    # Increase the timeout proportionally to the number of processors.
    ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
    if not $maxTimeout.nil?
        # The above adjustment can push the effective timeout over the timeout
        # enforced by buildbot's ShellCommand. Tests that hang would then result
        # in the whole run getting killed. Allow the user to cap the effective
        # timeout.
        if $maxTimeout < ENV["JSCTEST_timeout"].to_i
            ENV["JSCTEST_timeout"] = $maxTimeout.to_s
        end
    end
end

# We do not adjust hardTimeout. If we are not producing any results during 1200 seconds, buildbot terminates the tests. So we should terminate hung tests.

if !ENV["JSCTEST_memoryLimit"] && $memoryLimited
    ENV["JSCTEST_memoryLimit"] = (1 * 1024 * 1024 * 1024).to_s
end

# Some tests fail if the time zone is not set to US/Pacific
# https://webkit.org/b/136363
# Set as done in run-javascript-tests
ENV["TZ"] = "US/Pacific"; 

# Have crash reporter show the test command in Application Specific Information.
ENV["JSCTEST_CrashReportArgV"] = "1"

def runBundle
    raise unless $bundle

    testRunner = TestRunner.create($testRunnerType, $runnerDir)
    evaluator = BundleTestsExecutor.new($runlist, $serialPlans, $exclusivePlans, ITERATION_LIMITS, $treatFailingAsFlaky, testRunner)
    evaluator.loop
end

def runTarball
    raise unless $tarball

    prepareBundle
    testRunner = TestRunner.create($testRunnerType, $runnerDir)
    testRunner.prepare($runlist, $serialPlans, $exclusivePlans, Set.new, nil)
    compressBundle
end

def forEachRemote(remoteHosts, options={}, &blk)
    threads = []
    remoteHosts.each_index {
        | index |
        remoteHost = remoteHosts[index]
        threads << Thread.new {
            blk.call(index, remoteHost)
        }
    }

    etime = nil
    if options.has_key?(:timeout)
        etime = Time.now + options[:timeout]
    end
    liveRemotes = []
    threads.each_index {
        | index |
        thread = threads[index]
        begin
            if options.has_key?(:timeout)
                if etime.nil?
                    # If a timeout has been requested and etime is nil,
                    # that means the timeout has expired and we shouldn't
                    # wait at all.
                    timeout = 0
                else
                    timeout = etime - Time.now
                    if timeout < 0
                        timeout = 0
                    end
                end
                if thread.join(timeout).nil?
                    if $verbosity > 0
                        $stderr.puts("Timeout joining thread for remote #{remoteHosts[index]}")
                    end
                    # Timeout expired, so we can't block waiting for
                    # any other threads. Either they're done or
                    # they've also timed out.
                    etime = nil
                    raise CommandExecutionFailed
                end
            else
                thread.join # No timeout requested, just block.
            end
            liveRemotes << remoteHosts[index]
        rescue CommandExecutionFailed
            if options[:dropOnFailure]
                if $verbosity > 0
                    $stderr.puts("Dropping failed remote #{remoteHosts[index]}")
                end
            else
                raise
            end
        end
    }
    liveRemotes
end

def each_chunk(e, chunkSize)
    accumulator = []
    e.each { |element|
        accumulator << element
        if accumulator.size == chunkSize
            yield accumulator
            accumulator = []
        end
    }
    if accumulator.size > 0
        yield accumulator
    end
end

class TestRunnerGnuParallel < TestRunner
    def prepareGnuParallelRunnerJobs(name, runlist, completedPlans)
        path = @runnerDir + name
        FileUtils.mkdir_p(@runnerDir)

        File.open(path, "w") {
            | outp |
            runlist = runlist.select { |plan|
                not completedPlans.include?(plan)
            }
            each_chunk(runlist, $gnuParallelChunkSize) { |plans|
                job = plans.collect { |plan|
                    "sh ./test_script_#{plan.index}"
                }.join("; ")
                outp.puts(job)
            }
        }
    end
    def prepareRunner(runlist, serialPlans, exclusivePlans, completedPlans, remoteHosts)
        prepareGnuParallelRunnerJobs("parallel-tests", runlist, completedPlans | serialPlans | exclusivePlans)
        prepareGnuParallelRunnerJobs("serial-tests", serialPlans | exclusivePlans, completedPlans)
    end
end


def withGnuParallelSshWrapper(&blk)
    Tempfile.open('ssh-wrapper', $runnerDir) {
        | wrapper |
        head =
<<'EOF'
#!/bin/sh

remotedir="$1"
shift

remoteport="$1"
shift

remoteuser="$1"
shift

remotehost="$1"
shift

if test "x$1" != "x--"; then
   echo "Expected '--' at this position, instead got $1" 1>&2
   exit 3
fi
shift
EOF
        extraOptions = [
            # Many of our jobs are short, ensure GNU parallel won't need to set
            # up an ssh connection from scratch.
            "ControlPath=./%C",
            "ControlMaster=auto",
            "ControlPersist=10m",
            # Treat remote boards as volatile. Don't check the host keys, don't
            # save them locally.
            "StrictHostKeyChecking=no",
            "UserKnownHostsFile=/dev/null"
        ].collect { |opt|
            ["-o", opt]
        }.flatten
        sshOptions = (SSH_OPTIONS_DEFAULT + extraOptions).join(" ")
        wrapper.puts(head +
                     "echo \"$@\" | ssh #{sshOptions} -p \"$remoteport\" -l \"$remoteuser\" -o RemoteCommand=\"if test -d '$remotedir'; then cd '$remotedir'; else echo '#{PARALLEL_REMOTE_WRAPPER_MARK_BEGIN}${remotehost}#{PARALLEL_REMOTE_WRAPPER_MARK_END}'; false; fi && sh -s\" \"$remotehost\""
        )
        FileUtils.chmod("ugo=rx", wrapper.path)
        wrapper.close # Avoid ETXTBUSY
        blk.call(wrapper.path)
    }
end

def withGnuParallelSshLoginFile(remoteHosts, &blk)
    withGnuParallelSshWrapper {
        | wrapper |
        Tempfile.open('slf', $runnerDir) {
            | tf |
            remoteHosts.each {
                | remoteHost |
                tf.puts("#{wrapper} #{remoteHost.remoteDirectory} #{remoteHost.port} #{remoteHost.user} #{remoteHost.host}")
            }
            tf.flush
            blk.call(tf.path)
        }
    }
end

def unpackBundleOnRemoteHosts(remoteHosts)
    forEachRemote(remoteHosts, :dropOnFailure => true, :timeout => REMOTE_TIMEOUT) {
        | _, remoteHost |
        mysys(["ssh"] + SSH_OPTIONS_DEFAULT +
               (remoteHost.identity_file_path ? ["-i", remoteHost.identity_file_path] : []) +
               ["-p", remoteHost.port.to_s,
               "#{remoteHost.user}@#{remoteHost.host}",
               "cd #{Shellwords.shellescape(remoteHost.remoteDirectory)} && rm -rf #{$outputDir.basename} && tar xzf #{$tarFileName}"])
    }
end

def prepareRemotes(remoteHosts)
    # If the preparatory steps fail, drop the remote host from our
    # list. Otherwise, if it comes back online in the middle of an
    # iteration, we'll try to run test jobs on it, possibly using
    # an unrelated bundle from a previous run.
    remoteHosts = forEachRemote(remoteHosts, {:dropOnFailure => true, :timeout => REMOTE_TIMEOUT}) {
        | _, remoteHost |
        getRemoteDirectoryIfNeeded(remoteHost)
        copyBundleToRemote(remoteHost)
    }
    return unpackBundleOnRemoteHosts(remoteHosts)
end

def runGnuParallelRunner(remoteHosts, inputs, options={})
    timeout = 300
    if ENV["JSCTEST_timeout"]
        timeout = ENV["JSCTEST_timeout"].to_f.ceil.to_i
    end
    # Keep ncpus + 1 jobs running by default to avoid any stalls due
    # to ssh latency.
    parallelJobsOnEachHost = "+1"
    if $numChildProcessesSetByUser
        parallelJobsOnEachHost = $numChildProcesses
    end
    if options[:parallelJobsOnEachHost]
      parallelJobsOnEachHost = options[:parallelJobsOnEachHost]
    end
    markerWithHost = Regexp.new(".*#{PARALLEL_REMOTE_WRAPPER_MARK_BEGIN}(.*)#{PARALLEL_REMOTE_WRAPPER_MARK_END}.*")
    markerWithoutHost = Regexp.new(".*#{PARALLEL_REMOTE_STATE_LOST_MARKER}.*")
    withGnuParallelSshLoginFile(remoteHosts) {
        | slf |
        cmd = [
            "parallel",
            "-j", "#{parallelJobsOnEachHost}",
            # NB: the tests exit with 0 regardless of whether they
            # passed or failed, so this will only retry tests that we
            # weren't able to get a result for, likely because the
            # connection went down or the remote OOM'd/crashed).
            "--retries", "5",
            "--line-buffer", # we know our output is line-oriented
            "--slf", slf,
            "--timeout", timeout.to_s,
            "-a", inputs,
            "if test -e #{$outputDir.basename}/.runner; then cd #{$outputDir.basename}/.runner; else echo #{PARALLEL_REMOTE_STATE_LOST_MARKER}; false; fi && " +
            exportBaseEnvironmentVariables(false) +
            $envVars.collect { |var | "export #{var} &&"}.join("") +
            "sh -c "
        ]
        runAndMonitorCommandOutput(cmd) {
            | pid, line |
            host = "<unknown host>"
            md = markerWithoutHost.match(line)
            if md.nil?
                md = markerWithHost.match(line)
                host = md[1] unless md.nil?
            end
            if not md.nil?
                if $verbosity > 0
                    $stderr.puts("Remote host lost state, triggering high-level retry: #{host}")
                end
                # We could try to reprovision this specific remote
                # host, but that seems needlessly complicated (we
                # don't expect the remotes to go down every
                # minute...). Simply kill the GNU parallel process.
                Process.kill("TERM", pid)
            end
        }
    }
end

# Unconditionally run the selftests on every execution, they're pretty cheap.
ExecutorSelfTests.run
TestResultEvaluatorSelfTests.run

class NonRemoteTestsExecutor < BasicTestsExecutor
    def initialize(runlist, serialPlans, exclusivePlans, maxIterationsBounds, treatFailingAsFlaky, testRunner)
        super(runlist, serialPlans, exclusivePlans,
              nil,
              maxIterationsBounds,
              treatFailingAsFlaky
             )
        @testRunner = testRunner
    end
    def prepareExecution(remoteHosts)
        remoteHosts
    end
    def refreshExecution(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
        if not @remoteHosts.nil?
            raise "remoteHosts #{@remoteHosts}, expected nil"
        end
        @testRunner.prepare(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
    end
end

class NormalTestsExecutor < NonRemoteTestsExecutor
    def prepareArtifacts(remoteHosts)
        raise "remoteHosts not nil" unless remoteHosts.nil?
        prepareBundle
        @testRunner.prepare(@runlist, @serialPlans, @exclusivePlans, Set.new, nil)
    end
    def executeTests(remoteHosts)
        runTestRunner(@testRunner, nil, nil)
        cleanEmptyResultFiles
    end
    def updateStatusMap(iteration, statusMap)
        return getStatusMap(statusMap)
    end
end

class BundleTestsExecutor < NonRemoteTestsExecutor
    def prepareArtifacts(remoteHosts)
        raise "remoteHosts not nil" unless remoteHosts.nil?
        cleanRunnerDirectory
        cleanOldResults
    end
    def executeTests(remoteHosts)
        runTestRunner(@testRunner, nil, nil)
        cleanEmptyResultFiles
    end
    def updateStatusMap(iteration, statusMap)
        return getStatusMap(statusMap)
    end
end

class BaseRemoteTestsExecutor < BasicTestsExecutor
    def initialize(runlist, serialPlans, exclusivePlans, remoteHosts, iterationLimits,
                   treatFailingAsFlaky, testRunner)
        super(runlist, serialPlans, exclusivePlans, remoteHosts, iterationLimits, treatFailingAsFlaky)
        @testRunner = testRunner
    end
    def updateStatusMap(iteration, statusMap)
        return getStatusMap(statusMap)
    end
    def refreshExecution(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
        # Note, when running tests remotely, we always prepare tests
        # for all the hosts when refreshing execution, as even hosts
        # that went away during testing may have come back online in
        # the meantime -- so we don't expect a list of live remote
        # hosts to be passed in.
        @testRunner.prepare(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
    end
end

class RemoteTestsExecutor < BaseRemoteTestsExecutor
    def prepareArtifacts(remoteHosts)
        raise "remoteHosts nil" if remoteHosts.nil?
        prepareBundle

        # The make runner statically shards the tests, so do a
        # liveness check before @testRunner.prepare. Otherwise, we'd
        # always end up scheduling tests on dead remotes and hamper
        # our progress.
        remoteHosts = getLiveRemoteHosts(remoteHosts)
        @testRunner.prepare(@runlist, @serialPlans, @exclusivePlans, Set.new, remoteHosts)
        compressBundle
        remoteHosts
    end
    def prepareExecution(remoteHosts)
        return prepareRemotes(remoteHosts)
    end
    def refreshExecution(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
        # Don't assign tests to remotes that are down.
        remoteHosts = getLiveRemoteHosts(remoteHosts)
        super(runlist, serialPlans, exclusivePlans, completedTests, remoteHosts)
    end
    def executeTests(remoteHosts)
        forEachRemote(remoteHosts, :dropOnFailure => true) {
            | index |
            runTestRunner(@testRunner, remoteHosts, index)
        }
    end
    private
    def getLiveRemoteHosts(remoteHosts)
        # This command is supposed to work on all remote hosts, so
        # reuse it for the liveness check.
        cmd = commandToGetNumberOfProcessors
        forEachRemote(remoteHosts, :dropOnFailure => true, :timeout => REMOTE_TIMEOUT) { |_, remoteHost|
            begin
                sshRead(cmd, remoteHost)
            rescue
                # It's what forEachRemote considers an expected failure.
                raise CommandExecutionFailed
            end
        }
    end
end

class GnuParallelTestsExecutor < BaseRemoteTestsExecutor
    def prepareArtifacts(remoteHosts)
        prepareBundle
        @testRunner.prepare(@runlist, @serialPlans, @exclusivePlans, Set.new, remoteHosts)
        compressBundle
    end
    def prepareExecution(remoteHosts)
        return prepareRemotes(remoteHosts)
    end
    def executeTests(remoteHosts)
        runGnuParallelRunner(remoteHosts, $runnerDir + "serial-tests",
                             { :parallelJobsOnEachHost => 1})
        runGnuParallelRunner(remoteHosts, $runnerDir + "parallel-tests")
    end
end

def runNormal
    raise if $bundle or $tarball

    testRunner = TestRunner.create($testRunnerType, $runnerDir)
    evaluator = NormalTestsExecutor.new($runlist, $serialPlans, $exclusivePlans, ITERATION_LIMITS,
                                        $treatFailingAsFlaky, testRunner)
    statusMap = evaluator.loop
    detectFailures(statusMap)

end

def runRemote
    raise unless $remote

    testRunner = TestRunner.create($testRunnerType, $runnerDir)
    executor = RemoteTestsExecutor.new($runlist, $serialPlans, $exclusivePlans, $remoteHosts, ITERATION_LIMITS, $treatFailingAsFlaky, testRunner)
    statusMap = executor.loop
    detectFailures(statusMap)
end

def runGnuParallel
    raise "Can only use --gnu-parallel-runner with --remote*" unless $remote
    raise "Can only use --gnu-parallel-runner with the default writer" unless $testWriter == "default"
    testRunner = TestRunner.create($testRunnerType, $runnerDir)
    executor = GnuParallelTestsExecutor.new($runlist, $serialPlans, $exclusivePlans,
                                            $remoteHosts, ITERATION_LIMITS,
                                            $treatFailingAsFlaky, testRunner)
    statusMap = executor.loop
    detectFailures(statusMap)
end

puts

if $testRunnerType == :gnuparallel
    raise unless $remote
end

if $bundle
    runBundle
elsif $remote
    if $testRunnerType == :gnuparallel
        runGnuParallel
    else
        runRemote
    end
elsif $tarball
    runTarball
else
    runNormal
end