test/ci.testone.cmd

::-------------------------------------------------------------------------------------------------------
:: Copyright (C) Microsoft. All rights reserved.
:: Copyright (c) 2021 ChakraCore Project Contributors. All rights reserved.
:: Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
::-------------------------------------------------------------------------------------------------------

:: ============================================================================
::
:: testone.cmd
::
:: Runs tests for continuous integration. This script is called from
:: the CI build and it runs all tests for x86 and x64, debug and test
:: build configs.
::
:: Do not use this script to run all tests on your dev box.
::   - It will delete all your existing test logs
::   - It does not run the various flavors of the tests in parallel (though
::     this is not currently possible anyway because rl stages logs in a
::     common directory)
::   - It does nothing to provide useful output when there are failures, e.g.
::     they can be buried under thousands of lines of output from further
::     tests run.
::   - It cannot be cancelled without risk of polluting your command prompt
::     environment with environment variables that will make further calls to
::     runtests.cmd behave unexpectedly.
::
:: ============================================================================

@echo off
setlocal

REM check that we have enough parameters
if "%1"=="" (
    goto :usage
)
if "%2"=="" (
    goto :usage
)

if "%_ENTRY_SCRIPT_NAME%"=="" (
    set _ENTRY_SCRIPT_NAME=%0
)

set _RootDir=%~dp0..
set _HadFailures=0
set _error=0

:: ============================================================================
:: Main script
:: ============================================================================
:main

  pushd %_RootDir%\test
  set _TestDir=%CD%

  call ci.parsetestargs.cmd %*

  set _LogDir=%_TestDir%\logs\%_TestArch%_%_TestType%
  set _TestArgs=%_TestArch%%_TestType%
  set _BinDir=%_RootDir%\Build\VcBuild%_SpecialBuild%\bin

  call :doSilent rd /s/q %_LogDir%

  call :verifyBytcode

  if "%_HadFailures%" == "0" (
    call :runTests %_TestArgs%

    if "%_ReducedTestRun%" == "1" (
      echo -- ci.testone.cmd ^>^> Reduced test run: skipping native tests.
    ) else (
      call :runNativeTests %_TestArgs%
    )
  )

  echo.
  echo -- ci.testone.cmd ^>^> Failure code: %_HadFailures%
  if "%_HadFailures%" NEQ "0" (
    if "%_HadFailures%" == "2" (
      echo -- ci.testone.cmd ^>^> Bytecode test failed, bytecode needs to be updated! 1>&2
    ) else if "%_HadFailures%" == "3" (
      echo -- ci.testone.cmd ^>^> Unit tests failed! 1>&2
    ) else if "%_HadFailures%" == "4" (
      echo -- ci.testone.cmd ^>^> Native tests failed! 1>&2
      call :summarizeLogs
    ) else (
      echo -- ci.testone.cmd ^>^> Unknown failure! 1>&2
    )
  ) else (
    echo -- ci.testone.cmd ^>^> Tests passed!
  )

  popd

  exit /b %_HadFailures%

  goto :eof

:: ============================================================================
:: Verify that generated bytecode if up to date for one config report if not
:: ============================================================================
:verifyBytcode

  set bytecode_type=--jit
  if "%_SpecialBuild%" == ".NoJIT" (
    set bytecode_type=--noJit
  )

  call :do python %_RootDir%\tools\regenByteCode.py --verify %bytecode_type% --%_TestArch% --binary=%_BinDir%\%_TestArch%_%_TestType%\ch.exe

  if "%_error%" NEQ "0" (
    echo -- ci.testone.cmd ^>^> verifying bytecode failed
    set _HadFailures=2
  )

  goto :eof

:: ============================================================================
:: Run one test suite against one build config and record if there were errors
:: ============================================================================
:runTests

  if "%_SpecialBuild%" == ".NoJIT" (
    set OverideVariant=--variants=disable_jit
  )

  call :do python %_TestDir%\runtests.py --%_TestArch% --%_TestType% %OverideVariant% %_ExtraTestArgs% --binary=%_BinDir%\%_TestArch%_%_TestType%\ch.exe

  if "%_error%" NEQ "0" (
    echo -- ci.testone.cmd ^>^> runtests.py failed
    set _HadFailures=3
  )

  goto :eof

:: ============================================================================
:: Run jsrt test suite against one build config and record if there were errors
:: ============================================================================
:runNativeTests

  echo -- ci.testone.cmd ^>^> Running native tests... this can take some time
  if not exist %_LogDir%\ mkdir %_LogDir%
  set _LogFile=%_LogDir%\nativetests.log
  call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% -d yes > %_LogFile% 2>&1
  echo -- ci.testone.cmd ^>^> Running native tests... DONE!

  if "%_error%" NEQ "0" (
    echo -- ci.testone.cmd ^>^> runnativetests.cmd failed; printing %_LogFile%
    powershell "if (Test-Path %_LogFile%) { Get-Content %_LogFile% }"
    set _HadFailures=4
  )

  goto :eof

:: ============================================================================
:: Summarize the logs into a listing of only the failures
:: ============================================================================
:summarizeLogs

  pushd %_LogDir%
  findstr /sip failed nativetests.log >> summary.log
  rem Echo to stderr so that VSO includes the output in the build summary

  type summary.log 1>&2
  popd

:: ============================================================================
:: Echo a command line before executing it
:: ============================================================================
:do

  echo -- ci.testone.cmd ^>^> :do %*
  cmd /s /c "%*"
  set _error=%ERRORLEVEL%

  goto :eof

:: ============================================================================
:: Echo a command line before executing it and redirect the command's output
:: to nul
:: ============================================================================
:doSilent

  echo -- ci.testone.cmd ^>^> :doSilent %* ^> nul 2^>^&1
  cmd /s /c "%* > nul 2>&1"
  set _error=%ERRORLEVEL%

  goto :eof


:: ============================================================================
:: Not enough params
:: ============================================================================
:usage

    echo Not enough parameters. Please specify architecture and type.
    echo Examples:
    echo.
    echo     %_ENTRY_SCRIPT_NAME% x86 debug
    echo     %_ENTRY_SCRIPT_NAME% x86 test
    echo.
    echo     %_ENTRY_SCRIPT_NAME% x64 debug
    echo     %_ENTRY_SCRIPT_NAME% x64 test

    goto :end