.circleci/config.yml

version: 2.1

orbs:
  win: circleci/windows@1.0.0

executors:
  linux-node:
    docker:
      - image: circleci/node:stretch
  bionic:
    docker:
      - image: emscripten/emscripten-ci
    environment:
      LANG: "C.UTF-8"
      EMTEST_DETECT_TEMPFILE_LEAKS: "1"
      EMCC_CORES: "4"
      EMSDK_NOTTY: "1"
      EMTEST_WASI_SYSROOT: "~/wasi-sdk/wasi-sysroot"
  mac:
    environment:
      EMSDK_NOTTY: "1"
    macos:
      xcode: "12.2.0"

commands:
  download-chrome:
    description: "Download chrome"
    steps:
      - run:
          name: download chrome
          command: |
            # Using stable rather than beta until we can fix
            # Currently downloading form our own buckets due to:
            # https://github.com/emscripten-core/emscripten/issues/14987
            #wget -O ~/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
            wget -O ~/chrome.deb https://storage.googleapis.com/webassembly/chrome/google-chrome-stable_current_amd64.deb
            dpkg -i ~/chrome.deb
  emsdk-env:
    description: "emsdk_env.sh"
    steps:
      - run:
          name: emsdk_env.sh
          command: echo "source ~/emsdk/emsdk_env.sh 2> /dev/null" >> $BASH_ENV
  npm-install:
    description: "npm ci"
    steps:
      - run: npm ci
  build:
    description: "Install emsdk"
    steps:
      - run:
          name: install emsdk
          command: |
            curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz
            tar -C ~ -xf ~/emsdk-main.tar.gz
            mv ~/emsdk-main ~/emsdk
            cd ~/emsdk
            ./emsdk update-tags
            ./emsdk install tot-upstream
            ./emsdk activate tot-upstream
            # Remove the emsdk version of emscripten to save space in the
            # persistent workspace and to avoid any confusion with the version
            # we are trying to test.
            rm -Rf emscripten
            echo "import os" >> .emscripten
            # We use an out-of-tree cache directory so it can be part of the
            # persisted workspace (see below).
            echo "CACHE = os.path.expanduser('~/cache')" >> .emscripten
            # test v8 with --wasm-staging which enables new features. we use v8
            # as a "bleeding edge" shell here, basically, with node being the
            # primary shell (which we run without any special flags)
            # Refer to commit 0bc3640 if we need to pin V8 version.
            echo "V8_ENGINE = [os.path.expanduser('~/.jsvu/v8'), '--wasm-staging']" >> .emscripten
            echo "JS_ENGINES = [NODE_JS]" >> .emscripten
            echo "if os.path.exists(V8_ENGINE[0]): JS_ENGINES.append(V8_ENGINE)" >> .emscripten
            echo "WASM_ENGINES = []" >> .emscripten
            test -f ~/vms/wasmtime && echo "WASMTIME = os.path.expanduser(os.path.join('~', 'vms', 'wasmtime')) ; WASM_ENGINES.append(WASMTIME)" >> .emscripten || true
            test -f ~/vms/wasmer && echo "WASMER = os.path.expanduser(os.path.join('~', 'vms', 'wasmer')) ; WASM_ENGINES.append(WASMER)" >> .emscripten || true
            test -d ~/wasi-sdk && cp -ar ~/wasi-sdk/lib/ ~/emsdk/upstream/lib/clang/14.0.0/
            cd -
            echo "final .emscripten:"
            cat ~/emsdk/.emscripten
      - emsdk-env
      - npm-install
      - run:
          name: clear cache
          command: |
            ./emcc --clear-cache
  build-libs-and-freeze:
    description: "Build all libraries, and freeze the cache"
    steps:
      - run:
          name: embuilder build ALL
          command: |
            ./embuilder build ALL
            ./tests/runner test_hello_world
      - run:
          name: embuilder (LTO)
          command: |
            ./embuilder build MINIMAL --lto
            ./tests/runner test_hello_world
      - run:
          name: embuilder (PIC)
          command: |
            ./embuilder build SYSTEM --pic
            ./tests/runner test_hello_world
      - run:
          name: embuilder (PIC+LTO)
          command: |
            ./embuilder build MINIMAL --pic --lto
            ./tests/runner test_hello_world
      - run:
          name: freeze cache
          command: |
            echo "FROZEN_CACHE=True" >> ~/emsdk/.emscripten
  persist:
    description: "Persist the emsdk, libraries, and engines"
    steps:
      - persist_to_workspace:
          # Must be an absolute path, or relative path from working_directory
          root: ~/
          # Must be relative path from root
          paths:
            - emsdk/
            - cache/
            - vms/
            - wasi-sdk/
            - .jsvu/
  run-tests:
    description: "Runs emscripten tests"
    parameters:
      test_targets:
        description: "Test suites to run"
        type: string
    steps:
      - attach_workspace:
          # Must be absolute path or relative path from working_directory
          at: ~/
      - checkout
      - run:
          name: submodule update
          command: git submodule update --init
      - emsdk-env
      - npm-install
      - run:
          name: run tests
          command: |
            ./tests/runner << parameters.test_targets >>
            $EMSDK_PYTHON ./tools/check_clean.py
  run-tests-mac:
    description: "Runs emscripten tests"
    parameters:
      test_targets:
        description: "Test suites to run"
        type: string
    steps:
      - attach_workspace:
          # Must be absolute path or relative path from working_directory
          at: ~/
      - checkout
      - emsdk-env
      - npm-install
      - run:
          name: Install brew package dependencies
          environment:
            HOMEBREW_NO_AUTO_UPDATE: "1"
          command: |
            brew list cmake || brew install cmake
            brew list python3 || brew install python3
      - run:
          name: run tests
          command: |
            ./tests/runner << parameters.test_targets >>
            $EMSDK_PYTHON ./tools/check_clean.py
  test-firefox:
    description: "Runs emscripten tests under firefox"
    steps:
      - attach_workspace:
          # Must be absolute path or relative path from working_directory
          at: ~/
      - checkout
      - run:
          name: submodule update
          command: git submodule update --init
      - emsdk-env
      - npm-install
      - run:
          name: download firefox
          command: |
            wget -O ~/ff.tar.bz2 "https://download.mozilla.org/?product=firefox-devedition-latest-ssl&os=linux64&lang=en-US"
            tar -C ~ -xf ~/ff.tar.bz2
      - run:
          name: configure firefox
          command: |
            mkdir ~/tmp-firefox-profile/
            cat > ~/tmp-firefox-profile/user.js \<<EOF
            user_pref("gfx.offscreencanvas.enabled", true);
            user_pref("javascript.options.shared_memory", true);
            user_pref("dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", true);
            EOF
      - run:
          # browser.test_sdl2_mouse and/or SDL2 should be fixed. The case happens
          # to be failing here, and the root cause might be related with the
          # initial position of the mouse pointer relative to the canvas.
          # browser.test_html5_webgl_create_context is skipped because
          # anti-aliasing is not well supported.
          # browser.test_webgl_offscreen_canvas_in_pthread and
          # browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread
          # are crashing Firefox (bugzil.la/1281796). The former case is
          # further blocked by issue #6897.
          # browser.test_sdl2_misc_main_module: Requires PIC version of libSDL
          # which is not included in emsdk (not compatible with FROZEN_CACHE).
          name: run tests
          environment:
            GALLIUM_DRIVER: softpipe # TODO: use the default llvmpipe when it supports more extensions
            # TODO: Do GL testing when https://bugzil.la/1375585 (lack of WebGL
            #       support in headless mode) resolves
            EMTEST_LACKS_GRAPHICS_HARDWARE: "1"
            EMTEST_LACKS_SOUND_HARDWARE: "1"
            # OffscreenCanvas support is not yet done in Firefox.
            EMTEST_LACKS_OFFSCREEN_CANVAS: "1"
            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
            DISPLAY: ":0"
          command: |
            export EMTEST_BROWSER="$HOME/firefox/firefox -headless -profile $HOME/tmp-firefox-profile/"
            echo "-----"
            echo "Running browser tests"
            echo "-----"
            tests/runner browser skip:browser.test_sdl2_mouse skip:browser.test_html5_webgl_create_context skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget skip:browser.test_sdl2_misc_main_module
            # posix and emrun suites are disabled because firefox errors on
            #  "Firefox is already running, but is not responding."
            # TODO: find out a way to shut down and restart firefox
  test-chrome:
    description: "Runs emscripten browser tests under chrome"
    steps:
      - attach_workspace:
          # Must be absolute path or relative path from working_directory
          at: ~/
      - checkout
      - run:
          name: submodule update
          command: git submodule update --init
      - emsdk-env
      - npm-install
      - download-chrome
      - run:
          name: run tests
          environment:
            EMTEST_LACKS_SOUND_HARDWARE: "1"
            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
            # --no-sandbox becasue we are running as root and chrome requires
            # this flag for now: https://crbug.com/638180
            CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile"
            CHROME_FLAGS_HEADLESS: "--headless --remote-debugging-port=1234"
            CHROME_FLAGS_WASM: "--enable-features=WebAssembly --enable-features=SharedArrayBuffer --disable-features=WebAssemblyTrapHandler --js-flags=\"--experimental-wasm-threads --harmony-sharedarraybuffer\""
            CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito"
          command: |
            export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
            # skip test_zzz_zzz_4gb_fail as it OOMs on the current bot
            # browser.test_sdl2_misc_main_module: Requires PIC version of libSDL
            # which is not included in emsdk (not compatible with FROZEN_CACHE).
            tests/runner browser posixtest_browser.test_pthread_create_1_1 skip:browser.test_zzz_zzz_4gb_fail skip:browser.test_sdl2_misc_main_module
            tests/runner emrun
  test-sockets-chrome:
    description: "Runs emscripten sockets tests under chrome"
    steps:
      - attach_workspace:
          # Must be absolute path or relative path from working_directory
          at: ~/
      - checkout
      - emsdk-env
      - npm-install
      - download-chrome
      - run:
          name: run sockets tests
          environment:
            EMTEST_LACKS_SOUND_HARDWARE: "1"
            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
            # --no-sandbox becasue we are running as root and chrome requires
            # this flag for now: https://crbug.com/638180
            CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile"
            CHROME_FLAGS_HEADLESS: "--headless --remote-debugging-port=1234"
            CHROME_FLAGS_WASM: "--enable-features=WebAssembly --enable-features=SharedArrayBuffer --disable-features=WebAssemblyTrapHandler --js-flags=\"--experimental-wasm-threads --harmony-sharedarraybuffer --no-wasm-disable-structured-cloning\""
            CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito"
          command: |
            export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
            tests/runner sockets

jobs:
  build-docs:
    executor: bionic
    steps:
      - checkout
      # This version is mentioned in `site/source/docs/site/about.rst`.
      # Please keep them in sync.
      - run: pip3 install sphinx==2.4.4
      - run: make -C site text
      - run: tests/check_emcc_help_text.py
      - run: make -C site html
  flake8:
    executor: bionic
    steps:
      - checkout
      - run: pip3 install -r requirements-dev.txt
      - run: python3 -m flake8 --show-source --statistics
  eslint:
    executor: linux-node
    steps:
      - checkout
      - npm-install
      - run: npm run lint
  test-sanity:
    executor: bionic
    steps:
      - run-tests:
          test_targets: "sanity"
  build-linux:
    executor: bionic
    steps:
      - checkout
      - run:
          name: get wasmer
          command: |
            wget https://github.com/wasmerio/wasmer/releases/download/0.11.0/wasmer-linux-amd64.tar.gz
            tar -xf wasmer-linux-amd64.tar.gz
            mkdir ~/vms
            cp bin/wasmer ~/vms
      - run:
          name: get wasmtime
          command: |
            # use a pinned version due to https://github.com/bytecodealliance/wasmtime/issues/714
            export VERSION=v0.8.0
            wget https://github.com/bytecodealliance/wasmtime/releases/download/v0.8.0/wasmtime-$VERSION-x86_64-linux.tar.xz
            tar -xf wasmtime-$VERSION-x86_64-linux.tar.xz
            cp wasmtime-$VERSION-x86_64-linux/wasmtime ~/vms
      - run:
          name: get wasi-sdk-sysroot
          command: |
            wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/libclang_rt.builtins-wasm32-wasi-11.0.tar.gz
            wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sysroot-11.0.tar.gz
            mkdir ~/wasi-sdk
            tar xvf libclang_rt.builtins-wasm32-wasi-11.0.tar.gz -C ~/wasi-sdk
            tar xvf wasi-sysroot-11.0.tar.gz -C ~/wasi-sdk/
      - run:
          name: get v8
          command: |
            wget https://nodejs.org/dist/v12.13.0/node-v12.13.0-linux-x64.tar.xz
            tar -xf node-v12.13.0-linux-x64.tar.xz
            export PATH="`pwd`/node-v12.13.0-linux-x64/bin:${PATH}"
            npm install jsvu -g
            export PATH="${HOME}/.jsvu:${PATH}"
            jsvu --os=default --engines=v8
      - build
      - build-libs-and-freeze
      - persist
  # Perhaps we don't need to run this suite with every commit. Consider moving this to FYI bot.
  test-posixtest:
    executor: bionic
    steps:
      - run-tests:
          test_targets: "posixtest"
  test-wasm0:
    executor: bionic
    steps:
      - run-tests:
          test_targets: "wasm0"
  test-wasm2:
    executor: bionic
    steps:
      - run-tests:
          # also add a few asan tests
          test_targets: "wasm2 asan.test_embind* asan.test_abort_on_exceptions asan.test_ubsan_full_left_shift_fsanitize_integer asan.test_pthread* asan.test_dyncall_specific_minimal_runtime asan.test_async_hello"
  test-wasm3:
    executor: bionic
    steps:
      - run-tests:
          # also add a little select testing for wasm2js in -O3
          test_targets: "wasm3 wasm2js3.test_memorygrowth_2"
  test-wasm2js1:
    executor: bionic
    steps:
      - run-tests:
          test_targets: "wasm2js1"
  test-other:
    executor: bionic
    steps:
      - run-tests:
          # some native-dependent tests fail because of the lack of native
          # headers on emsdk-bundled clang
          test_targets: "other skip:other.test_native_link_error_message"
  test-browser-chrome:
    executor: bionic
    steps:
      - test-chrome
  test-browser-firefox:
    executor: bionic
    steps:
      - test-firefox
  test-sockets-chrome:
    executor: bionic
    steps:
      - test-sockets-chrome
  # windows and mac do not have separate build and test jobs, as they only run
  # a limited set of tests; it is simpler and faster to do it all in one job.
  test-windows:
    executor:
      name: win/vs2019
      shell: bash.exe -eo pipefail
    environment:
      PYTHONUNBUFFERED: "1"
      # Override the default locale to use UTF-8 encoding for all files and stdio 
      # streams (see PEP540), since oure test files are encoded with UTF-8.
      PYTHONUTF8: "1"
      EMSDK_NOTTY: "1"
      # clang can compile but not link in the current setup, see
      # https://github.com/emscripten-core/emscripten/pull/11382#pullrequestreview-428902638
      EMTEST_LACKS_NATIVE_CLANG: "1"
      EMTEST_SKIP_V8: "1"
    steps:
      - checkout
      - run:
          name: Install packages
          command: |
            choco install cmake.portable ninja
            choco install python --version 3.8.0
      - run:
          name: Add python to bash path
          command: echo "export PATH=\"$PATH:/c/python38/\"" >> $BASH_ENV
      - build
      # note we do *not* build all libraries and freeze the cache; as we run
      # only limited tests here, it's more efficient to build on demand
      - run-tests:
          test_targets: "other.test_emcc_cflags other.test_stdin other.test_bad_triple wasm2.test_sse1 wasm2.test_ccall other.test_closure_externs other.test_binaryen_debug other.test_js_optimizer_parse_error other.test_output_to_nowhere other.test_emcc_dev_null other.test_cmake* other.test_system_include_paths other.test_emar_response_file wasm2.test_utf16 other.test_special_chars_in_arguments other.test_toolchain_profiler"
  test-mac:
    executor: mac
    environment:
      EMTEST_SKIP_V8: "1"
    steps:
      - run:
          name: Install brew package dependencies
          environment:
            HOMEBREW_NO_AUTO_UPDATE: "1"
          command: |
            brew list cmake || brew install cmake
            brew list python3 || brew install python3
      - checkout
      - build
      # note we do *not* build all libraries and freeze the cache; as we run
      # only limited tests here, it's more efficient to build on demand
      - run-tests-mac:
          test_targets: "other wasm0.test_lua wasm0.test_longjmp_standalone wasm2.test_sse1 skip:other.test_native_link_error_message"

workflows:
  build-test:
    jobs:
      - flake8
      - eslint
      - build-docs
      - build-linux
      - test-sanity:
          requires:
            - build-linux
      - test-posixtest:
          requires:
            - build-linux
      - test-wasm0:
          requires:
            - build-linux
      - test-wasm2:
          requires:
            - build-linux
      - test-wasm3:
          requires:
            - build-linux
      - test-wasm2js1:
          requires:
            - build-linux
      - test-other:
          requires:
            - build-linux
      - test-browser-chrome:
          requires:
            - build-linux
      - test-browser-firefox:
          requires:
            - build-linux
      - test-sockets-chrome:
          requires:
            - build-linux
      - test-windows
      - test-mac