css/tools/travis/build.py

import itertools
import os
import shutil
import subprocess
import sys

import vcs

here = os.path.abspath(os.path.dirname(__file__))

source_dir = os.path.join(here, "..", "..")

remote_built = "https://github.com/jgraham/css-test-built.git"
built_dir = os.path.join(here, "css-test-built")

local_files = ["manifest", "serve", "serve.py", ".gitmodules", "tools", "resources",
               "config.default.json"]

def get_hgsubstate():
    state = {}
    with open(os.path.join(source_dir, ".hgsubstate"), "r") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            revision, path = line.split(" ", 1)
            state[path] = revision
    return state

def fetch_submodules():
    hg = vcs.hg
    orig_dir = os.getcwd()
    state = get_hgsubstate()
    for tool in ["apiclient", "w3ctestlib"]:
        dest_dir = os.path.join(source_dir, "tools", tool)
        repo_path = "tools/" + tool
        if os.path.exists(os.path.join(dest_dir, ".hg")):
            try:
                os.chdir(dest_dir)
                if repo_path in state:
                    rev = state[repo_path]
                    try:
                        hg("update", rev, log_error=False)
                    except subprocess.CalledProcessError:
                        hg("pull")
                        hg("update", rev)
                else:
                    hg("pull")
                    hg("update")
            finally:
                os.chdir(orig_dir)
        else:
            hg("clone", ("https://hg.csswg.org/dev/%s" % tool), dest_dir)
            try:
                os.chdir(dest_dir)
                if repo_path in state:
                    hg("update", state[repo_path])
                else:
                    hg("update")
            finally:
                os.chdir(orig_dir)

def update_dist():
    if not os.path.exists(built_dir) or not vcs.is_git_root(built_dir):
        git = vcs.git
        git("clone", "--depth", "1", remote_built, built_dir)
    else:
        git = vcs.bind_to_repo(vcs.git, built_dir)
        git("fetch")
        if "origin/master" in git("branch", "-a"):
            git("checkout", "master")
            git("merge", "--ff-only", "origin/master")

    git = vcs.bind_to_repo(vcs.git, built_dir)
    git("config", "user.email", "CssBuildBot@users.noreply.github.com")
    git("config", "user.name", "CSS Build Bot")
    git("submodule", "update", "--init", "--recursive")

def setup_virtualenv():
    virtualenv_path = os.path.join(here, "_virtualenv")

    if not os.path.exists(virtualenv_path):
        subprocess.check_call(["virtualenv", virtualenv_path])

    activate_path = os.path.join(virtualenv_path, "bin", "activate_this.py")

    execfile(activate_path, dict(__file__=activate_path))

    subprocess.check_call(["pip", "-q", "install", "mercurial"])
    subprocess.check_call(["pip", "-q", "install", "html5lib==0.9999999"])
    subprocess.check_call(["pip", "-q", "install", "lxml"])
    subprocess.check_call(["pip", "-q", "install", "Template-Python"])


def update_to_changeset(changeset):
    git = vcs.bind_to_repo(vcs.git, source_dir)
    git("checkout", changeset)
    apply_build_system_fixes()

def apply_build_system_fixes():
    fixes = [
        "c017547f65e07bdd889736524d47824d032ba2e8",
        "cb4a737a88aa7e2f4e54383c57ffa2dfae093dcf",
        "ec540343a3e729644c8178dbcf6d063dca20d49f",
    ]
    git = vcs.bind_to_repo(vcs.git, source_dir)
    for fix in fixes:
        git("cherry-pick", "--keep-redundant-commits", fix)

def build_tests():
    subprocess.check_call(["python", os.path.join(source_dir, "tools", "build.py")],
                           cwd=source_dir)

def remove_current_files():
    for node in os.listdir(built_dir):
        if node.startswith(".git"):
            continue

        if node in ("resources", "tools"):
            continue

        path = os.path.join(built_dir, node)
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)

def copy_files():
    dist_path = os.path.join(source_dir, "dist")
    for node in os.listdir(dist_path):
        src_path = os.path.join(dist_path, node)
        dest_path = os.path.join(built_dir, node)
        if os.path.isdir(src_path):
            shutil.copytree(src_path, dest_path)
        else:
            shutil.copy2(src_path, dest_path)

def update_git():
    git = vcs.bind_to_repo(vcs.git, built_dir)
    git("add", ".")

def add_changeset(changeset):
    git = vcs.bind_to_repo(vcs.git, built_dir)

    dest_path = os.path.join(built_dir, "source_rev")
    with open(dest_path, "w") as f:
        f.write(changeset)
    git("add", os.path.relpath(dest_path, built_dir))

def commit(changeset):
    git = vcs.git

    msg = git("log", "-r", changeset, "-n", "1", "--pretty=%B", repo=source_dir)
    msg = "%s\n\nBuild from revision %s" % (msg, changeset)

    git("commit", "-m", msg, repo=built_dir)

def get_new_commits():
    git = vcs.bind_to_repo(vcs.git, source_dir)
    commit_path = os.path.join(built_dir, "source_rev")
    with open(commit_path) as f:
        prev_commit = f.read().strip()

    if git("rev-parse", "--revs-only", prev_commit).strip() != prev_commit:
        # we don't have prev_commit in current tree, so let's just do what's new
        commit_range = os.environ['TRAVIS_COMMIT_RANGE']
        assert (os.environ["TRAVIS_PULL_REQUEST"] != "false" or
                os.environ["TRAVIS_BRANCH"] != "master")
    else:
        merge_base = git("merge-base", prev_commit, os.environ['TRAVIS_COMMIT']).strip()
        commit_range = "%s..%s" % (merge_base, os.environ['TRAVIS_COMMIT'])

    commits = git("log", "--pretty=%H", "-r", commit_range).strip()
    if not commits:
        return []
    return reversed(commits.split("\n"))

def maybe_push():
    if os.environ["TRAVIS_PULL_REQUEST"] != "false":
        return

    if os.environ["TRAVIS_BRANCH"] != "master":
        return

    git = vcs.bind_to_repo(vcs.git, built_dir)

    out = "https://%s@github.com/jgraham/css-test-built.git" % os.environ["TOKEN"]
    git("remote", "add", "out", out, quiet=True)

    for i in range(2):
        try:
            git("push", "out", "HEAD:master")
        except subprocess.CalledProcessError:
            if i == 0:
                git("fetch", "origin")
                git("rebase", "origin/master")
        else:
            return

    raise Exception("Push failed")

def main():
    setup_virtualenv()
    fetch_submodules()
    update_dist()
    changesets = list(get_new_commits())
    print >> sys.stderr, "Building %d changesets:" % len(changesets)
    print >> sys.stderr, "\n".join(changesets)
    if len(changesets) > 50:
        raise Exception("Building more than 50 changesets, giving up")

    for changeset in changesets:
        update_to_changeset(changeset)
        remove_current_files()
        build_tests()
        copy_files()
        update_git()
        add_changeset(changeset)
        commit(changeset)
    maybe_push()

if __name__ == "__main__":
    main()