// Copyright 2017 The Clspv Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifdef _MSC_VER
#pragma warning(push, 0)
#endif

#include <cassert>
#include <cstring>
#include <iomanip>
#include <list>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <tuple>
#include <unordered_map>
#include <unordered_set>
#include <utility>

#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/UniqueVector.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ValueSymbolTable.h"
#include "llvm/Pass.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Cloning.h"

// enable spv::HasResultAndType
#define SPV_ENABLE_UTILITY_CODE
#include "spirv/unified1/spirv.hpp"

#include "clspv/AddressSpace.h"
#include "clspv/Option.h"
#include "clspv/PushConstant.h"
#include "clspv/SpecConstant.h"
#include "clspv/spirv_c_strings.hpp"
#include "clspv/spirv_glsl.hpp"
#include "clspv/spirv_reflection.hpp"

#include "ArgKind.h"
#include "Builtins.h"
#include "ComputeStructuredOrder.h"
#include "ConstantEmitter.h"
#include "Constants.h"
#include "DescriptorCounter.h"
#include "Layout.h"
#include "NormalizeGlobalVariable.h"
#include "Passes.h"
#include "SpecConstant.h"
#include "Types.h"

#if defined(_MSC_VER)
#pragma warning(pop)
#endif

using namespace llvm;
using namespace clspv;
using namespace clspv::Builtins;
using namespace clspv::Option;
using namespace mdconst;

namespace {

cl::opt<std::string> TestOutFile("producer-out-file", cl::init("test.spv"),
                                 cl::ReallyHidden,
                                 cl::desc("SPIRVProducer testing output file"));

cl::opt<bool> ShowResourceVars("show-rv", cl::init(false), cl::Hidden,
                               cl::desc("Show resource variable creation"));

cl::opt<bool>
    ShowProducerIR("show-producer-ir", cl::init(false), cl::ReallyHidden,
                   cl::desc("Dump the IR at the start of SPIRVProducer"));

cl::opt<bool>
    NameBasicBlocks("name-basic-blocks", cl::init(false), cl::ReallyHidden,
                    cl::desc("Name SPIR-V basic blocks based on LLVM names"));

// These hacks exist to help transition code generation algorithms
// without making huge noise in detailed test output.
const bool Hack_generate_runtime_array_stride_early = true;

// The value of 1/pi.  This value is from MSDN
// https://msdn.microsoft.com/en-us/library/4hwaceh6.aspx
const double kOneOverPi = 0.318309886183790671538;

// SPIRV Module Sections (per 2.4 of the SPIR-V spec)
// These are used to collect SPIRVInstructions by type on-the-fly.
enum SPIRVSection {
  kCapabilities,
  kExtensions,
  kImports,
  kMemoryModel,
  kEntryPoints,
  kExecutionModes,

  kDebug,
  kNames,
  kAnnotations,

  kTypes,
  kConstants = kTypes,
  kGlobalVariables,

  kFunctions,

  // This is not a section of the SPIR-V spec and should always immediately
  // precede kSectionCount. It is a convenient place for the embedded
  // reflection data.
  kReflection,
  kSectionCount
};

class SPIRVID {
  uint32_t id;

public:
  SPIRVID(uint32_t _id = 0) : id(_id) {}
  uint32_t get() const { return id; }
  bool isValid() const { return id != 0; }
  bool operator==(const SPIRVID &that) const { return id == that.id; }
  bool operator!=(const SPIRVID &that) const { return id != that.id; }
  bool operator<(const SPIRVID &that) const { return id < that.id; }
};

enum SPIRVOperandType { NUMBERID, LITERAL_WORD, LITERAL_DWORD, LITERAL_STRING };

struct SPIRVOperand {
  SPIRVOperand(SPIRVOperandType Ty, uint32_t Num) : Type(Ty) {
    LiteralNum[0] = Num;
  }
  SPIRVOperand(SPIRVOperandType Ty, const char *Str)
      : Type(Ty), LiteralStr(Str) {}
  SPIRVOperand(SPIRVOperandType Ty, StringRef Str)
      : Type(Ty), LiteralStr(Str) {}
  explicit SPIRVOperand(ArrayRef<uint32_t> NumVec) {
    auto sz = NumVec.size();
    assert(sz >= 1 && sz <= 2);
    Type = sz == 1 ? LITERAL_WORD : LITERAL_DWORD;
    LiteralNum[0] = NumVec[0];
    if (sz == 2) {
      LiteralNum[1] = NumVec[1];
    }
  }

  SPIRVOperandType getType() const { return Type; }
  uint32_t getNumID() const { return LiteralNum[0]; }
  std::string getLiteralStr() const { return LiteralStr; }
  const uint32_t *getLiteralNum() const { return LiteralNum; }

  uint32_t GetNumWords() const {
    switch (Type) {
    case NUMBERID:
    case LITERAL_WORD:
      return 1;
    case LITERAL_DWORD:
      return 2;
    case LITERAL_STRING:
      // Account for the terminating null character.
      return uint32_t((LiteralStr.size() + 4) / 4);
    }
    llvm_unreachable("Unhandled case in SPIRVOperand::GetNumWords()");
  }

private:
  SPIRVOperandType Type;
  std::string LiteralStr;
  uint32_t LiteralNum[2];
};

typedef SmallVector<SPIRVOperand, 4> SPIRVOperandVec;

namespace {

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, const SPIRVID &v) {
  list.emplace_back(NUMBERID, v.get());
  return list;
}

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, uint32_t num) {
  list.emplace_back(LITERAL_WORD, num);
  return list;
}

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, int32_t num) {
  list.emplace_back(LITERAL_WORD, static_cast<uint32_t>(num));
  return list;
}

SPIRVOperandVec &operator<<(SPIRVOperandVec &list,
                            const std::vector<uint32_t> &num_vec) {
  list.emplace_back(num_vec);
  return list;
}

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, StringRef str) {
  list.emplace_back(LITERAL_STRING, str);
  return list;
}

} // namespace

struct SPIRVInstruction {
  // Primary constructor must have Opcode, initializes WordCount based on ResID.
  SPIRVInstruction(spv::Op Opc, SPIRVID ResID = 0)
      : Opcode(static_cast<uint16_t>(Opc)) {
    setResult(ResID);
  }

  // Creates an instruction with an opcode and no result ID, and with the given
  // operands.  This calls primary constructor to initialize Opcode, WordCount.
  // Takes ownership of the operands and clears |Ops|.
  SPIRVInstruction(spv::Op Opc, SPIRVOperandVec &Ops) : SPIRVInstruction(Opc) {
    setOperands(Ops);
  }
  // Creates an instruction with an opcode and no result ID, and with the given
  // operands.  This calls primary constructor to initialize Opcode, WordCount.
  // Takes ownership of the operands and clears |Ops|.
  SPIRVInstruction(spv::Op Opc, SPIRVID ResID, SPIRVOperandVec &Ops)
      : SPIRVInstruction(Opc, ResID) {
    setOperands(Ops);
  }

  uint32_t getWordCount() const { return WordCount; }
  uint16_t getOpcode() const { return Opcode; }
  SPIRVID getResultID() const { return ResultID; }
  const SPIRVOperandVec &getOperands() const { return Operands; }

private:
  void setResult(SPIRVID ResID = 0) {
    WordCount = 1 + (ResID.isValid() ? 1 : 0);
    ResultID = ResID;
  }

  void setOperands(SPIRVOperandVec &Ops) {
    assert(Operands.empty());
    Operands = std::move(Ops);
    for (auto &opd : Operands) {
      WordCount += uint16_t(opd.GetNumWords());
    }
  }

private:
  uint32_t WordCount; // Check the 16-bit bound at code generation time.
  uint16_t Opcode;
  SPIRVID ResultID;
  SPIRVOperandVec Operands;
};

struct SPIRVProducerPassImpl {

  // Struct to handle generation of ArrayStride decorations.
  struct StrideType {
    uint32_t stride;
    SPIRVID id;

    StrideType(uint32_t stride, SPIRVID id) : stride(stride), id(id) {}

    bool operator<(const StrideType &x) const {
      if (stride < x.stride)
        return true;
      if (x.stride < stride)
        return false;
      return id.get() < x.id.get();
    }
  };

  typedef DenseMap<Type *, SPIRVID> TypeMapType;
  typedef DenseMap<Type *, SmallVector<SPIRVID, 2>> LayoutTypeMapType;
  typedef UniqueVector<Type *> TypeList;
  typedef UniqueVector<StrideType> StrideTypeList;
  typedef DenseMap<Value *, SPIRVID> ValueMapType;
  typedef DenseMap<DIFile *, SPIRVID> DIFileMap;
  typedef DenseMap<BasicBlock *, std::pair<uint32_t, uint32_t>> BBDILocMap;
  typedef std::list<SPIRVID> SPIRVIDListType;
  typedef std::vector<std::pair<Value *, SPIRVID>> EntryPointVecType;
  typedef std::set<uint32_t> CapabilitySetType;
  typedef std::list<SPIRVInstruction> SPIRVInstructionList;
  typedef std::map<spv::BuiltIn, SPIRVID> BuiltinConstantMapType;
  // A vector of pairs, each of which is:
  // - the LLVM instruction that we will later generate SPIR-V code for
  // - the SPIR-V instruction placeholder that will be replaced
  typedef std::vector<std::pair<Value *, SPIRVInstruction *>>
      DeferredInstVecType;
  typedef DenseMap<FunctionType *, std::pair<FunctionType *, uint32_t>>
      GlobalConstFuncMapType;

  SPIRVProducerPassImpl(raw_pwrite_stream *out, bool outputCInitList,
                        ModuleAnalysisManager &MAM, bool *HasErrorOut,
                        std::string *OutputLog)
      : module(nullptr), MAM(&MAM), out(out),
        binaryTempOut(binaryTempUnderlyingVector), binaryOut(out),
        outputCInitList(outputCInitList), patchBoundOffset(0), nextID(1),
        OpExtInstImportID(0), HasVariablePointersStorageBuffer(false),
        HasVariablePointers(false), HasNonUniformPointers(false),
        HasConvertToF(false), HasIntegerDot(false), SamplerPointerTy(nullptr),
        SamplerDataTy(nullptr), WorkgroupSizeValueID(0), WorkgroupSizeVarID(0),
        TestOutput(out == nullptr), HasErrorOut(HasErrorOut),
        OutputLog(OutputLog), HasError(false) {
    addCapability(spv::CapabilityShader);
    if (clspv::Option::PhysicalStorageBuffers())
      addCapability(spv::CapabilityPhysicalStorageBufferAddresses);
    Ptr = this;
  }

  bool runOnModule(Module &module);

  // output the SPIR-V header block
  void outputHeader();

  // patch the SPIR-V header block
  void patchHeader();

  CapabilitySetType &getCapabilitySet() { return CapabilitySet; }
  TypeMapType &getImageTypeMap() { return ImageTypeMap; }
  ValueMapType &getValueMap() { return ValueMap; }
  DIFileMap &getDebugDIFileMap() { return DebugDIFileMap; }
  BBDILocMap &getBBDILocMap() { return DebugBBDILocMap; }
  SPIRVInstructionList &getSPIRVInstList(SPIRVSection Section) {
    return SPIRVSections[Section];
  };
  EntryPointVecType &getEntryPointVec() { return EntryPointVec; }
  DeferredInstVecType &getDeferredInstVec() { return DeferredInstVec; }
  SPIRVIDListType &getEntryPointInterfacesList() {
    return EntryPointInterfacesList;
  }
  SPIRVID getOpExtInstImportID();
  std::vector<SPIRVID> &getBuiltinDimVec() { return BuiltinDimensionVec; }

  bool hasVariablePointersStorageBuffer() {
    return HasVariablePointersStorageBuffer;
  }
  void setVariablePointersStorageBuffer() {
    if (!HasVariablePointersStorageBuffer) {
      addCapability(spv::CapabilityVariablePointersStorageBuffer);
      HasVariablePointersStorageBuffer = true;
    }
  }
  bool hasVariablePointers() { return HasVariablePointers; }
  void setVariablePointers() {
    if (!HasVariablePointers) {
      addCapability(spv::CapabilityVariablePointers);
      HasVariablePointers = true;
    }
  }
  bool hasNonUniformPointers() { return HasNonUniformPointers; }
  void setNonUniformPointers() {
    if (!HasNonUniformPointers) {
      addCapability(spv::CapabilityShaderNonUniform);
      HasNonUniformPointers = true;
    }
  }
  bool hasConvertToF() { return HasConvertToF; }
  void setConvertToF() {
    if (!HasConvertToF &&
        (ExecutionModeRoundingModeRTE(FloatingPointType::fp16) ||
         ExecutionModeRoundingModeRTE(FloatingPointType::fp32) ||
         ExecutionModeRoundingModeRTE(FloatingPointType::fp64))) {
      addCapability(spv::CapabilityRoundingModeRTE);
      HasConvertToF = true;
    }
  }
  bool hasIntegerDot() { return HasIntegerDot; }
  void setIntegerDot() { HasIntegerDot = true; }
  GlobalConstFuncMapType &getGlobalConstFuncTypeMap() {
    return GlobalConstFuncTypeMap;
  }
  SmallPtrSet<Value *, 16> &getGlobalConstArgSet() {
    return GlobalConstArgumentSet;
  }
  StrideTypeList &getTypesNeedingArrayStride() { return TypesNeedingArrayStride; }

  void ReadFunctionAttributes();

  void GenerateLLVMIRInfo();
  // Populate GlobalConstFuncTypeMap. Also, if module-scope __constant will
  // *not* be converted to a storage buffer, replace each such global variable
  // with one in the storage class expecgted by SPIR-V.
  void FindGlobalConstVars();
  // Populate ResourceVarInfoList, FunctionToResourceVarsMap, and
  // ModuleOrderedResourceVars.
  void FindResourceVars();
  void FindTypesForSamplerMap();
  void FindTypesForResourceVars();

  // Lookup or create pointer type.
  //
  // Returns the SPIRVID of the pointer type.
  // Return the pointer address space of |v|. If |v| is in
  // GlobalConstArgumentSet, then the address space is ModuleScopePrivate.
  unsigned GetPointerAddressSpace(Value *v) {
    if (v->getType()->isPointerTy()) {
      if (getGlobalConstArgSet().count(v)) {
        return AddressSpace::ModuleScopePrivate;
      }
      return v->getType()->getPointerAddressSpace();
    }
    return 0;
  }

  SPIRVID getSPIRVPointerType(Type *PtrTy, Type *DataTy);

  // Lookup or create function type.
  //
  // Returns the SPIRVID of the function type.
  // Defers to getSPIRVType if the function type doesn't use opaque pointer
  // types.
  SPIRVID getSPIRVFunctionType(FunctionType *FTy, Type *RetTy,
                               ArrayRef<Type *> ParamTys);

  // Returns the canonical type of |type|.
  //
  // By default, clspv maps both __constant and __global address space pointers
  // to StorageBuffer storage class. In order to prevent duplicate types from
  // being generated, clspv uses the canonical type as a representative.
  Type *CanonicalType(Type *type);

  // Lookup or create Types, Constants.
  // Returns SPIRVID once it has been created.
  SPIRVID getSPIRVType(Type *Ty, bool needs_layout);
  SPIRVID getSPIRVType(Type *Ty);
  SPIRVID getSPIRVConstant(Constant *Cst, Type *TyHint = nullptr);
  SPIRVID getSPIRVInt32Constant(uint32_t CstVal);
  SPIRVID getSPIRVInt64Constant(uint64_t CstVal);
  // Lookup SPIRVID of llvm::Value, may create Constant.
  SPIRVID getSPIRVValue(Value *V, Type *TyHint = nullptr);
  SPIRVID getSPIRVPointerOperand(Value *PtrVal, Type *ExpectedPointeeTy);

  bool PointerRequiresLayout(unsigned aspace);
  bool UntypedPointerStorageClass(spv::StorageClass sc);

  SPIRVID getSPIRVBuiltin(spv::BuiltIn BID, Type *Ty, spv::Capability Cap);

  void GenerateModuleInfo();
  void GenerateGlobalVar(GlobalVariable &GV);
  void GenerateWorkgroupVars();
  // Generate reflection instructions for resource variables associated with
  // arguments to F.
  void GenerateSamplers();
  // Generate OpVariables for %clspv.resource.var.* calls.
  void GenerateResourceVars();
  void GenerateFuncPrologue(Function &F);
  void GenerateFuncBody(Function &F);
  void GenerateEntryPointInitialStores();
  spv::Op GetSPIRVCmpOpcode(CmpInst *CmpI);
  spv::Op GetSPIRVPointerCmpOpcode(CmpInst *CmpI);
  spv::Op GetSPIRVCastOpcode(Instruction &I);
  spv::Op GetSPIRVBinaryOpcode(Instruction &I);
  SPIRVID GenerateClspvInstruction(CallInst *Call,
                                   const FunctionInfo &FuncInfo);
  SPIRVID GenerateImageInstruction(CallInst *Call,
                                   const FunctionInfo &FuncInfo);
  SPIRVID GenerateSubgroupInstruction(CallInst *Call,
                                      const FunctionInfo &FuncInfo);
  SPIRVID GenerateInstructionFromCall(CallInst *Call);
  SPIRVID GenerateShuffle2FromCall(Type *Ty, Value *SrcA, Value *SrcB,
                                   Value *Mask);
  SPIRVID GeneratePopcount(Type *Ty, Value *BaseValue, LLVMContext &Context);
  SPIRVID GenerateFabs(Value *Input);
  SPIRVID GenerateFabs(SPIRVID Input, Type *InputTy);
  SPIRVID GenerateCanonicalize(Value *val);
  SPIRVID GenerateCanonicalize(SPIRVID val, Type *InputTy);
  SPIRVID GenerateCopySignZero(SPIRVID val, Type *InputTy);
  bool ShouldFlushDenorms(Type *Ty) const;
  bool ShouldHackFMulFlushToZero(Type *Ty) const;
  SPIRVID GenerateIntegerDot(CallInst *Call, const FunctionInfo &func_info);
  void GenerateInstruction(Instruction &I);
  void GenerateFuncEpilogue();
  void HandleDeferredInstruction();
  void HandleDeferredDecorations();
  bool is4xi8vec(Type *Ty) const;
  spv::StorageClass GetStorageBufferClass() const;
  spv::StorageClass GetStorageClass(unsigned AddrSpace) const;
  spv::StorageClass GetStorageClassForArgKind(clspv::ArgKind arg_kind) const;
  spv::BuiltIn GetBuiltin(StringRef globalVarName) const;
  void WriteOneWord(uint32_t Word);
  void WriteResultID(const SPIRVInstruction &Inst);
  void WriteWordCountAndOpcode(const SPIRVInstruction &Inst);
  void WriteOperand(const SPIRVOperand &Op);
  void WriteSPIRVBinary();
  void WriteSPIRVBinary(SPIRVInstructionList &SPIRVInstList);

  // Returns true if |type| is compatible with OpConstantNull.
  bool IsTypeNullable(const Type *type) const;

  // Populate UBO remapped type maps.
  void PopulateUBOTypeMaps();

  // Populate the merge and continue block maps.
  void PopulateStructuredCFGMaps();

  // Wrapped methods of DataLayout accessors. If |type| was remapped for UBOs,
  // uses the internal map, otherwise it falls back on the data layout.
  uint64_t GetTypeSizeInBits(Type *type, const DataLayout &DL);
  uint64_t GetTypeAllocSize(Type *type, const DataLayout &DL);
  uint32_t GetExplicitLayoutStructMemberOffset(StructType *type,
                                               unsigned member,
                                               const DataLayout &DL);

  // Returns the base pointer of |v|.
  Value *GetBasePointer(Value *v);

  // Add Capability if not already (e.g. CapabilityGroupNonUniformBroadcast)
  void addCapability(uint32_t c) { CapabilitySet.emplace(c); }

  // Sets |HasVariablePointersStorageBuffer| or |HasVariablePointers| base on
  // |address_space|.
  void setVariablePointersCapabilities(unsigned address_space);

  // Returns true if |lhs| and |rhs| represent the same resource or workgroup
  // variable.
  bool sameResource(Value *lhs, Value *rhs) const;

  // Returns true if |inst| is phi or select that selects from the same
  // structure (or null).
  bool selectFromSameObject(Instruction *inst);

  bool isPointerUniform(Value *ptr);

  // Returns true if |Arg| is called with a coherent resource.
  bool CalledWithCoherentResource(Argument &Arg);

  bool NeedDecorationNoContraction(spv::Op op) {
    static const SmallVector<spv::Op> list_full = {
        spv::OpFMul,     spv::OpFDiv,        spv::OpFNegate,
        spv::OpFAdd,     spv::OpFSub,        spv::OpFRem,
        spv::OpFConvert, spv::OpConvertUToF, spv::OpConvertSToF};
    static const SmallVector<spv::Op> list_mad_enable = {
        spv::OpFDiv,     spv::OpFNegate,     spv::OpFSub,       spv::OpFRem,
        spv::OpFConvert, spv::OpConvertUToF, spv::OpConvertSToF};
    auto check_list = [&op](const SmallVector<spv::Op> &list) {
      for (auto opf : list) {
        if (op == opf)
          return true;
      }
      return false;
    };
    if (clspv::Option::ClMadEnable()) {
      return check_list(list_mad_enable);
    } else {
      return check_list(list_full);
    }
  }

  uint32_t getCorrectedStride(uint32_t CurrStride) {
    if (CurrStride == 1 && !Int8Support()) {
      return 4;
    }
    return CurrStride;
  }

  //
  // Primary interface for adding SPIRVInstructions to a SPIRVSection.
  template <enum SPIRVSection TSection = kFunctions>
  SPIRVID addSPIRVInst(spv::Op Opcode, SPIRVOperandVec &Operands) {
    bool has_result, has_result_type;
    spv::HasResultAndType(Opcode, &has_result, &has_result_type);
    SPIRVID RID = has_result ? incrNextID() : 0;
    if (has_result && has_result_type && !Operands.empty() &&
        Operands[0].getType() == NUMBERID) {
      IDTypeMap[RID.get()] = SPIRVID(Operands[0].getNumID());
    }
    SPIRVSections[TSection].emplace_back(Opcode, RID, Operands);

    if (NeedDecorationNoContraction(Opcode) && !Option::UnsafeMath()) {
      SPIRVOperandVec Ops;
      Ops << RID << spv::DecorationNoContraction;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
    }
    return RID;
  }
  template <enum SPIRVSection TSection = kFunctions>
  SPIRVID addSPIRVInst(spv::Op Op) {
    SPIRVOperandVec Ops;
    return addSPIRVInst<TSection>(Op, Ops);
  }
  template <enum SPIRVSection TSection = kFunctions>
  SPIRVID addSPIRVInst(spv::Op Op, uint32_t V) {
    SPIRVOperandVec Ops;
    Ops.emplace_back(LITERAL_WORD, V);
    return addSPIRVInst<TSection>(Op, Ops);
  }
  template <enum SPIRVSection TSection = kFunctions>
  SPIRVID addSPIRVInst(spv::Op Op, const char *V) {
    SPIRVOperandVec Ops;
    Ops.emplace_back(LITERAL_STRING, V);
    return addSPIRVInst<TSection>(Op, Ops);
  }

  //
  // Add placeholder for llvm::Value that references future values.
  // Must have result ID just in case final SPIRVInstruction requires.
  SPIRVID addSPIRVPlaceholder(Value *I) {
    SPIRVID RID = incrNextID();
    SPIRVOperandVec Ops;
    SPIRVSections[kFunctions].emplace_back(spv::OpExtInst, RID, Ops);
    DeferredInstVec.push_back({I, &SPIRVSections[kFunctions].back()});
    return RID;
  }
  // Replace placeholder with actual SPIRVInstruction on the final pass
  // (HandleDeferredInstruction).
  SPIRVID replaceSPIRVInst(SPIRVInstruction *I, spv::Op Opcode,
                           SPIRVOperandVec &Operands) {
    bool has_result, has_result_type;
    spv::HasResultAndType(Opcode, &has_result, &has_result_type);
    SPIRVID RID = has_result ? I->getResultID() : 0;
    if (has_result && has_result_type && !Operands.empty() &&
        Operands[0].getType() == NUMBERID) {
      IDTypeMap[RID.get()] = SPIRVID(Operands[0].getNumID());
    }
    *I = SPIRVInstruction(Opcode, RID, Operands);
    return RID;
  }

  //
  // Add global variable and capture entry point interface
  SPIRVID addSPIRVGlobalVariable(const SPIRVID &TypeID, spv::StorageClass SC,
                                 const SPIRVID &DataTypeID = SPIRVID(),
                                 const SPIRVID &InitID = SPIRVID(),
                                 bool add_interface = false);

  SPIRVID getReflectionImport();
  void GenerateReflection();
  void GenerateKernelReflection();
  void GeneratePrintfReflection();
  void GeneratePushConstantReflection();
  void GenerateSpecConstantReflection();
  void GenerateWorkgroupSizeReflection();
  void AddArgumentReflection(const Function &F, SPIRVID kernel_decl,
                             const std::string &name, clspv::ArgKind arg_kind,
                             uint32_t ordinal, uint32_t descriptor_set,
                             uint32_t binding, uint32_t offset, uint32_t size,
                             uint32_t spec_id, uint32_t elem_size);

  SPIRVID ChangeLayout(SPIRVID object, Type *type, bool removeLayout);

private:
  std::unique_ptr<Module> module;

  // Set of Capabilities required
  CapabilitySetType CapabilitySet;

  // Map from clspv::BuiltinType to SPIRV Global Variable
  BuiltinConstantMapType BuiltinConstantMap;

  ModuleAnalysisManager *MAM;
  raw_pwrite_stream *out;

  // TODO(dneto): Wouldn't it be better to always just emit a binary, and then
  // convert to other formats on demand?

  // When emitting a C initialization list, the WriteSPIRVBinary method
  // will actually write its words to this vector via binaryTempOut.
  SmallVector<char, 100> binaryTempUnderlyingVector;
  raw_svector_ostream binaryTempOut;

  // Binary output writes to this stream, which might be |out| or
  // |binaryTempOut|.  It's the latter when we really want to write a C
  // initializer list.
  raw_pwrite_stream *binaryOut;
  const bool outputCInitList; // If true, output look like {0x7023, ... , 5}
  uint64_t patchBoundOffset;
  uint32_t nextID;

  SPIRVID incrNextID() { return nextID++; }

  // ID for OpTypeInt 32 1.
  SPIRVID int32ID;
  // ID for OpTypeVector %int 4.
  SPIRVID v4int32ID;

  DenseMap<Value *, Type *> InferredTypeCache;
  DenseMap<unsigned, SPIRVID> IDTypeMap;
  std::unordered_map<unsigned, LayoutTypeMapType> PointerTypeMap;
  TypeMapType FunctionTypeMap;

  // Maps an LLVM Value pointer to the corresponding SPIR-V Id.
  LayoutTypeMapType TypeMap;
  // Maps an LLVM image type to its SPIR-V ID.
  TypeMapType ImageTypeMap;
  // A unique-vector of LLVM types that map to a SPIR-V type.
  TypeList Types;
  // Maps an LLVM Value pointer to the corresponding SPIR-V Id.
  ValueMapType ValueMap;
  DIFileMap DebugDIFileMap;
  BBDILocMap DebugBBDILocMap;
  SPIRVInstructionList SPIRVSections[kSectionCount];

  EntryPointVecType EntryPointVec;
  DeferredInstVecType DeferredInstVec;
  SPIRVIDListType EntryPointInterfacesList;
  SPIRVID OpExtInstImportID;
  std::vector<SPIRVID> BuiltinDimensionVec;
  bool HasVariablePointersStorageBuffer;
  bool HasVariablePointers;
  std::set<Value *> NonUniformPointers;
  bool HasNonUniformPointers;
  bool HasConvertToF;
  bool HasIntegerDot;
  Type *SamplerPointerTy;
  Type *SamplerDataTy;
  DenseMap<unsigned, SPIRVID> SamplerLiteralToIDMap;
  DenseSet<uint32_t> DecoratedNonUniform;

  // If a function F has a pointer-to-__constant parameter, then this variable
  // will map F's type to (G, index of the parameter), where in a first phase
  // G is F's type.
  // TODO(dneto): This doesn't seem general enough?  A function might have
  // more than one such parameter.
  GlobalConstFuncMapType GlobalConstFuncTypeMap;
  SmallPtrSet<Value *, 16> GlobalConstArgumentSet;
  // An ordered set of pointer types of Base arguments to OpPtrAccessChain,
  // or array types, and which point into transparent memory (StorageBuffer
  // storage class).  These will require an ArrayStride decoration.
  // See SPV_KHR_variable_pointers rev 13.
  StrideTypeList TypesNeedingArrayStride;

  // This is truly ugly, but works around what look like driver bugs.
  // For get_local_size, an earlier part of the flow has created a module-scope
  // variable in Private address space to hold the value for the workgroup
  // size.  Its intializer is a uint3 value marked as builtin WorkgroupSize.
  // When this is present, save the IDs of the initializer value and variable
  // in these two variables.  We only ever do a vector load from it, and
  // when we see one of those, substitute just the value of the intializer.
  // This mimics what Glslang does, and that's what drivers are used to.
  // TODO(dneto): Remove this once drivers are fixed.
  SPIRVID WorkgroupSizeValueID;
  SPIRVID WorkgroupSizeVarID;

  bool TestOutput;
  bool *HasErrorOut;
  std::string *OutputLog;
  bool HasError;

  // Bookkeeping for mapping kernel arguments to resource variables.
  struct ResourceVarInfo {
    ResourceVarInfo(int index_arg, unsigned set_arg, unsigned binding_arg,
                    Function *fn, clspv::ArgKind arg_kind_arg, int coherent_arg,
                    Type *type)
        : index(index_arg), descriptor_set(set_arg), binding(binding_arg),
          var_fn(fn), arg_kind(arg_kind_arg), coherent(coherent_arg),
          data_type(type),
          addr_space(type->isPointerTy() ? type->getPointerAddressSpace() : 0) {
    }
    const int index; // Index into ResourceVarInfoList
    const unsigned descriptor_set;
    const unsigned binding;
    Function *const var_fn; // The @clspv.resource.var.* function.
    const clspv::ArgKind arg_kind;
    const int coherent;
    Type *data_type;
    const unsigned addr_space; // The LLVM address space
    // The SPIR-V ID of the OpVariable.  Not populated at construction time.
    SPIRVID var_id;
  };
  // A list of resource var info.  Each one correponds to a module-scope
  // resource variable we will have to create.  Resource var indices are
  // indices into this vector.
  SmallVector<std::unique_ptr<ResourceVarInfo>, 8> ResourceVarInfoList;
  // This is a vector of pointers of all the resource vars, but ordered by
  // kernel function, and then by argument.
  UniqueVector<ResourceVarInfo *> ModuleOrderedResourceVars;
  // Map a function to the ordered list of resource variables it uses, one for
  // each argument.  If an argument does not use a resource variable, it
  // will have a null pointer entry.
  using FunctionToResourceVarsMapType =
      DenseMap<Function *, SmallVector<ResourceVarInfo *, 8>>;
  FunctionToResourceVarsMapType FunctionToResourceVarsMap;

  // What LLVM types map to SPIR-V types needing layout?  These are the
  // arrays and structures supporting storage buffers and uniform buffers.
  TypeList TypesNeedingLayout;
  // What LLVM struct types map to a SPIR-V struct type with Block decoration?
  UniqueVector<StructType *> StructTypesNeedingBlock;
  // For a call that represents a load from an opaque type (samplers, images),
  // map it to the variable id it should load from.
  DenseMap<CallInst *, SPIRVID> ResourceVarDeferredLoadCalls;

  // An ordered list of the kernel arguments of type pointer-to-local.
  using LocalArgList = SmallVector<Argument *, 8>;
  LocalArgList LocalArgs;
  // Information about a pointer-to-local argument.
  struct LocalArgInfo {
    // The SPIR-V ID of the array variable.
    SPIRVID variable_id;
    // The element type of the
    Type *elem_type;
    // The ID of the array type.
    SPIRVID array_size_id;
    // The ID of the array type.
    SPIRVID array_type_id;
    // The ID of the pointer to the array type.
    SPIRVID ptr_array_type_id;
    // The specialization constant ID of the array size.
    int spec_id;
  };
  // A mapping from Argument to its assigned SpecId.
  DenseMap<const Argument *, int> LocalArgSpecIds;
  // A mapping from SpecId to its LocalArgInfo.
  DenseMap<int, LocalArgInfo> LocalSpecIdInfoMap;
  // A mapping from a remapped type to its real offsets.
  DenseMap<Type *, std::vector<uint32_t>> RemappedUBOTypeOffsets;
  // A mapping from a remapped type to its real sizes.
  DenseMap<Type *, std::tuple<uint64_t, uint64_t, uint64_t>>
      RemappedUBOTypeSizes;

  // Maps basic block to its merge block.
  DenseMap<BasicBlock *, BasicBlock *> MergeBlocks;
  // Maps basic block to its continue block.
  DenseMap<BasicBlock *, BasicBlock *> ContinueBlocks;

  SPIRVID ReflectionID;
  DenseMap<Function *, SPIRVID> KernelDeclarations;

  StringMap<std::string> functionAttrStrings;

  DenseMap<uint32_t, uint32_t> WorkgroupVariableSizeMap;

public:
  static SPIRVProducerPassImpl *Ptr;
};

} // namespace

SPIRVProducerPassImpl *SPIRVProducerPassImpl::Ptr = nullptr;

namespace {

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, Type *t) {
  list.emplace_back(NUMBERID,
                    SPIRVProducerPassImpl::Ptr->getSPIRVType(t).get());
  return list;
}

SPIRVOperandVec &operator<<(SPIRVOperandVec &list, Value *v) {
  list.emplace_back(NUMBERID,
                    SPIRVProducerPassImpl::Ptr->getSPIRVValue(v).get());
  return list;
}

} // namespace

void SPIRVProducerPassImpl::ReadFunctionAttributes() {
  auto md_node =
      module->getNamedMetadata(clspv::EntryPointAttributesMetadataName());
  if (md_node) {
    for (auto *operand : md_node->operands()) {
      auto key = cast<MDString>(operand->getOperand(0).get())->getString();
      auto value = cast<MDString>(operand->getOperand(1).get())->getString();
      functionAttrStrings[key] = value;
    }
  }
}

PreservedAnalyses SPIRVProducerPass::run(Module &M,
                                         ModuleAnalysisManager &MAM) {
  SPIRVProducerPassImpl impl(out, outputCInitList, MAM, has_error, output_log);
  impl.runOnModule(M);
  PreservedAnalyses PA;
  return PA;
}

bool SPIRVProducerPassImpl::runOnModule(Module &M) {
  // TODO(sjw): Need to reset all data members for each Module, or better
  // yet create a new SPIRVProducer for every module.. For now only
  // allow 1 call.
  assert(module == nullptr);
  module = CloneModule(M);
  if (ShowProducerIR) {
    llvm::outs() << *module << "\n";
  }

  SmallVector<char, 10000> *binary = nullptr;
  if (TestOutput) {
    binary = new SmallVector<char, 10000>();
    out = new raw_svector_ostream(*binary);
  }

  binaryOut = outputCInitList ? &binaryTempOut : out;

  NonUniformPointers.clear();

  ReadFunctionAttributes();
  PopulateUBOTypeMaps();
  PopulateStructuredCFGMaps();

  // SPIR-V always begins with its header information
  outputHeader();

  // Gather information from the LLVM IR that we require.
  GenerateLLVMIRInfo();

  // Collect information on global variables too.
  for (GlobalVariable &GV : module->globals()) {
    // If the GV is one of our special __spirv_* variables, remove the
    // initializer as it was only placed there to force LLVM to not throw the
    // value away.
    if (GV.getName().starts_with("__spirv_") ||
        GV.getAddressSpace() == clspv::AddressSpace::PushConstant) {
      GV.setInitializer(nullptr);
    }
  }

  // Generate literal samplers if necessary.
  GenerateSamplers();

  // Generate SPIRV variables.
  for (GlobalVariable &GV : module->globals()) {
    GenerateGlobalVar(GV);
  }
  GenerateResourceVars();
  GenerateWorkgroupVars();

  // Generate SPIRV instructions for each function.
  for (Function &F : *module) {
    if (F.isDeclaration()) {
      continue;
    }
    // We do not wish to deal with any constexpr
    BitcastUtils::RemoveCstExprFromFunction(&F);
    // Generate Function Prologue.
    GenerateFuncPrologue(F);

    // Generate SPIRV instructions for function body.
    GenerateFuncBody(F);

    // Generate Function Epilogue.
    GenerateFuncEpilogue();
  }

  HandleDeferredInstruction();
  if (HasError) {
    *HasErrorOut = true;
    return false;
  }
  HandleDeferredDecorations();

  // Generate SPIRV module information.
  GenerateModuleInfo();

  // Generate embedded reflection information.
  GenerateReflection();

  WriteSPIRVBinary();

  // We need to patch the SPIR-V header to set bound correctly.
  patchHeader();

  if (outputCInitList) {
    bool first = true;
    std::ostringstream os;

    auto emit_word = [&os, &first](uint32_t word) {
      if (!first)
        os << ",\n";
      os << word;
      first = false;
    };

    os << "{";
    const std::string str(binaryTempOut.str());
    for (unsigned i = 0; i < str.size(); i += 4) {
      const uint32_t a = static_cast<unsigned char>(str[i]);
      const uint32_t b = static_cast<unsigned char>(str[i + 1]);
      const uint32_t c = static_cast<unsigned char>(str[i + 2]);
      const uint32_t d = static_cast<unsigned char>(str[i + 3]);
      emit_word(a | (b << 8) | (c << 16) | (d << 24));
    }
    os << "}\n";
    *out << os.str();
  }

  if (TestOutput) {
    std::error_code error;
    raw_fd_ostream test_output(TestOutFile, error, llvm::sys::fs::FA_Write);
    test_output << static_cast<raw_svector_ostream *>(out)->str();
    delete out;
    delete binary;
  }

  return false;
}

void SPIRVProducerPassImpl::outputHeader() {
  binaryOut->write(reinterpret_cast<const char *>(&spv::MagicNumber),
                   sizeof(spv::MagicNumber));
  uint32_t minor = 0;
  switch (SpvVersion()) {
  case SPIRVVersion::SPIRV_1_0:
    minor = 0;
    break;
  case SPIRVVersion::SPIRV_1_3:
    minor = 3;
    break;
  case SPIRVVersion::SPIRV_1_4:
    minor = 4;
    break;
  case SPIRVVersion::SPIRV_1_5:
    minor = 5;
    break;
  case SPIRVVersion::SPIRV_1_6:
    minor = 6;
    break;
  default:
    llvm_unreachable("unhandled spir-v version");
    break;
  }
  uint32_t version = (1 << 16) | (minor << 8);
  binaryOut->write(reinterpret_cast<const char *>(&version), sizeof(version));

  // use Google's vendor ID
  const uint32_t vendor = 21 << 16;
  binaryOut->write(reinterpret_cast<const char *>(&vendor), sizeof(vendor));

  // we record where we need to come back to and patch in the bound value
  patchBoundOffset = binaryOut->tell();

  // output a bad bound for now
  binaryOut->write(reinterpret_cast<const char *>(&nextID), sizeof(nextID));

  // output the schema (reserved for use and must be 0)
  const uint32_t schema = 0;
  binaryOut->write(reinterpret_cast<const char *>(&schema), sizeof(schema));
}

void SPIRVProducerPassImpl::patchHeader() {
  // for a binary we just write the value of nextID over bound
  binaryOut->pwrite(reinterpret_cast<char *>(&nextID), sizeof(nextID),
                    patchBoundOffset);
}

void SPIRVProducerPassImpl::GenerateLLVMIRInfo() {
  // This function generates LLVM IR for function such as global variable for
  // argument, constant and pointer type for argument access. These information
  // is artificial one because we need Vulkan SPIR-V output. This function is
  // executed ahead of FindType and FindConstant.

  FindGlobalConstVars();

  FindResourceVars();

  FindTypesForSamplerMap();
  FindTypesForResourceVars();
}

void SPIRVProducerPassImpl::FindGlobalConstVars() {
  clspv::NormalizeGlobalVariables(*module);
  const DataLayout &DL = module->getDataLayout();

  SmallVector<GlobalVariable *, 8> GVList;
  SmallVector<GlobalVariable *, 8> DeadGVList;
  for (GlobalVariable &GV : module->globals()) {
    if (GV.getType()->getAddressSpace() == AddressSpace::Constant) {
      if (GV.use_empty() &&
          GV.getName() != clspv::ClusteredConstantsVariableName()) {
        DeadGVList.push_back(&GV);
      } else {
        GVList.push_back(&GV);
      }
    }
  }

  // Remove dead global __constant variables.
  for (auto GV : DeadGVList) {
    GV->eraseFromParent();
  }
  DeadGVList.clear();

  if (clspv::Option::ModuleConstantsInStorageBuffer()) {
    // For now, we only support a single storage buffer.
    if (!GVList.empty()) {
      assert(GVList.size() == 1);
      const auto *GV = GVList[0];
      const auto constants_byte_size =
          (GetTypeSizeInBits(GV->getInitializer()->getType(), DL)) / 8;
      const size_t kConstantMaxSize = 65536;
      if (constants_byte_size > kConstantMaxSize) {
        outs() << "Max __constant capacity of " << kConstantMaxSize
               << " bytes exceeded: " << constants_byte_size << " bytes used\n";
        llvm_unreachable("Max __constant capacity exceeded");
      }
    }
  } else {
    // Change global constant variable's address space to ModuleScopePrivate.
    auto &GlobalConstFuncTyMap = getGlobalConstFuncTypeMap();
    for (auto GV : GVList) {
      // Create new gv with ModuleScopePrivate address space.
      Type *NewGVTy = GV->getValueType();
      GlobalVariable *NewGV = new GlobalVariable(
          *module, NewGVTy, false, GV->getLinkage(), GV->getInitializer(), "",
          nullptr, GV->getThreadLocalMode(), AddressSpace::ModuleScopePrivate);
      NewGV->takeName(GV);

      const SmallVector<User *, 8> GVUsers(GV->user_begin(), GV->user_end());
      SmallVector<User *, 8> CandidateUsers;

      auto record_called_function_type_as_user =
          [&GlobalConstFuncTyMap](Value *gv, CallInst *call) {
            // Find argument index.
            unsigned index = 0;
            for (unsigned i = 0; i < call->arg_size(); i++) {
              if (gv == call->getOperand(i)) {
                // TODO(dneto): Should we break here?
                index = i;
              }
            }

            // Record function type with global constant.
            GlobalConstFuncTyMap[call->getFunctionType()] =
                std::make_pair(call->getFunctionType(), index);
          };

      for (User *GVU : GVUsers) {
        if (CallInst *Call = dyn_cast<CallInst>(GVU)) {
          record_called_function_type_as_user(GV, Call);
        } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(GVU)) {
          // Check GEP users.
          for (User *GEPU : GEP->users()) {
            if (CallInst *GEPCall = dyn_cast<CallInst>(GEPU)) {
              record_called_function_type_as_user(GEP, GEPCall);
            }
          }
        }

        CandidateUsers.push_back(GVU);
      }

      for (User *U : CandidateUsers) {
        // Update users of gv with new gv.
        if (!isa<Constant>(U)) {
          // #254: Can't change operands of a constant, but this shouldn't be
          // something that sticks around in the module.
          U->replaceUsesOfWith(GV, NewGV);
        }
      }

      // Delete original gv.
      GV->eraseFromParent();
    }
  }
}

void SPIRVProducerPassImpl::FindResourceVars() {
  ResourceVarInfoList.clear();
  FunctionToResourceVarsMap.clear();
  ModuleOrderedResourceVars.reset();
  // Normally, there is one resource variable per clspv.resource.var.*
  // function, since that is unique'd by arg type and index.  By design,
  // we can share these resource variables across kernels because all
  // kernels use the same descriptor set.
  //
  // But if the user requested distinct descriptor sets per kernel, then
  // the descriptor allocator has made different (set,binding) pairs for
  // the same (type,arg_index) pair.  Since we can decorate a resource
  // variable with only exactly one DescriptorSet and Binding, we are
  // forced in this case to make distinct resource variables whenever
  // the same clspv.resource.var.X function is seen with disintct
  // (set,binding) values.
  const bool always_distinct_sets =
      clspv::Option::DistinctKernelDescriptorSets();
  for (Function &F : *module) {
    // Rely on the fact the resource var functions have a stable ordering
    // in the module.
    if (Builtins::Lookup(&F) == Builtins::kClspvResource) {
      // Find all calls to this function with distinct set and binding pairs.
      // Save them in ResourceVarInfoList.

      // Determine uniqueness of the (set,binding) pairs only withing this
      // one resource-var builtin function.
      using SetAndBinding = std::pair<unsigned, unsigned>;
      // Maps set and binding to the resource var info.
      DenseMap<SetAndBinding, ResourceVarInfo *> set_and_binding_map;
      bool first_use = true;
      for (auto &U : F.uses()) {
        if (auto *call = dyn_cast<CallInst>(U.getUser())) {
          const auto set =
              unsigned(dyn_cast<ConstantInt>(
                           call->getArgOperand(
                               clspv::ClspvOperand::kResourceDescriptorSet))
                           ->getZExtValue());
          const auto binding = unsigned(
              dyn_cast<ConstantInt>(
                  call->getArgOperand(clspv::ClspvOperand::kResourceBinding))
                  ->getZExtValue());
          const auto arg_kind = clspv::ArgKind(
              dyn_cast<ConstantInt>(
                  call->getArgOperand(clspv::ClspvOperand::kResourceArgKind))
                  ->getZExtValue());
          const auto arg_index = unsigned(
              dyn_cast<ConstantInt>(
                  call->getArgOperand(clspv::ClspvOperand::kResourceArgIndex))
                  ->getZExtValue());
          const auto coherent = unsigned(
              dyn_cast<ConstantInt>(
                  call->getArgOperand(clspv::ClspvOperand::kResourceCoherent))
                  ->getZExtValue());
          const auto data_type =
              call->getArgOperand(clspv::ClspvOperand::kResourceDataType)
                  ->getType();

          // Find or make the resource var info for this combination.
          ResourceVarInfo *rv = nullptr;
          if (always_distinct_sets) {
            // Make a new resource var any time we see a different
            // (set,binding) pair.
            SetAndBinding key{set, binding};
            auto where = set_and_binding_map.find(key);
            if (where == set_and_binding_map.end()) {
              rv = new ResourceVarInfo(
                  static_cast<int>(ResourceVarInfoList.size()), set, binding,
                  &F, arg_kind, coherent, data_type);
              ResourceVarInfoList.emplace_back(rv);
              set_and_binding_map[key] = rv;
            } else {
              rv = where->second;
            }
          } else {
            // The default is to make exactly one resource for each
            // clspv.resource.var.* function.
            if (first_use) {
              first_use = false;
              rv = new ResourceVarInfo(
                  static_cast<int>(ResourceVarInfoList.size()), set, binding,
                  &F, arg_kind, coherent, data_type);
              ResourceVarInfoList.emplace_back(rv);
            } else {
              rv = ResourceVarInfoList.back().get();
            }
          }

          // Now populate FunctionToResourceVarsMap.
          auto &mapping =
              FunctionToResourceVarsMap[call->getParent()->getParent()];
          while (mapping.size() <= arg_index) {
            mapping.push_back(nullptr);
          }
          mapping[arg_index] = rv;
        }
      }
    }
  }

  // Populate ModuleOrderedResourceVars.
  for (Function &F : *module) {
    auto where = FunctionToResourceVarsMap.find(&F);
    if (where != FunctionToResourceVarsMap.end()) {
      for (auto &rv : where->second) {
        if (rv != nullptr) {
          ModuleOrderedResourceVars.insert(rv);
        }
      }
    }
  }
  if (ShowResourceVars) {
    for (auto *info : ModuleOrderedResourceVars) {
      outs() << "MORV index " << info->index << " (" << info->descriptor_set
             << "," << info->binding << ") " << *(info->var_fn->getReturnType())
             << " (" << *(info->data_type) << ")"
             << "\n";
    }
  }
}

void SPIRVProducerPassImpl::FindTypesForSamplerMap() {
  // If we are using a sampler map, find the type of the sampler.
  if (auto *F = module->getFunction(clspv::LiteralSamplerFunction())) {
    SamplerDataTy = F->getArg(ClspvOperand::kSamplerDataType)->getType();
    SamplerPointerTy =
        PointerType::get(module->getContext(), AddressSpace::UniformConstant);
  }
}

void SPIRVProducerPassImpl::FindTypesForResourceVars() {
  // Record types so they are generated.
  TypesNeedingLayout.reset();
  StructTypesNeedingBlock.reset();

  for (const auto *info : ModuleOrderedResourceVars) {
    Type *type = info->var_fn->getReturnType();

    switch (info->arg_kind) {
    case clspv::ArgKind::Buffer:
    case clspv::ArgKind::BufferUBO:
      if (auto *sty = dyn_cast<StructType>(info->data_type)) {
        StructTypesNeedingBlock.insert(sty);
      } else {
        errs() << *type << "\n";
        llvm_unreachable("Buffer arguments must map to structures!");
      }
      break;
    case clspv::ArgKind::Pod:
    case clspv::ArgKind::PodUBO:
    case clspv::ArgKind::PointerUBO:
    case clspv::ArgKind::PodPushConstant:
      if (auto *sty = dyn_cast<StructType>(info->data_type)) {
        StructTypesNeedingBlock.insert(sty);
      } else {
        errs() << *type << "\n";
        llvm_unreachable("POD arguments must map to structures!");
      }
      break;
    case clspv::ArgKind::SampledImage:
    case clspv::ArgKind::StorageImage:
    case clspv::ArgKind::Sampler:
    case clspv::ArgKind::StorageTexelBuffer:
    case clspv::ArgKind::UniformTexelBuffer:
      break;
    default:
      break;
    }
  }

  // If module constants are clustered in a storage buffer then that struct
  // needs layout decorations.
  if (clspv::Option::ModuleConstantsInStorageBuffer()) {
    for (GlobalVariable &GV : module->globals()) {
      PointerType *PTy = cast<PointerType>(GV.getType());
      const auto AS = PTy->getAddressSpace();
      const bool module_scope_constant_external_init =
          (AS == AddressSpace::Constant) && GV.hasInitializer();
      const spv::BuiltIn BuiltinType = GetBuiltin(GV.getName());
      if (module_scope_constant_external_init &&
          spv::BuiltInMax == BuiltinType) {
        StructTypesNeedingBlock.insert(
            cast<StructType>(GV.getValueType()));
      }
    }
  }

  for (const GlobalVariable &GV : module->globals()) {
    if (GV.getAddressSpace() == clspv::AddressSpace::PushConstant) {
      auto Ty = GV.getValueType();
      assert(Ty->isStructTy() && "Push constants have to be structures.");
      auto STy = cast<StructType>(Ty);
      StructTypesNeedingBlock.insert(STy);
    } else if (GV.getName() == clspv::PrintfBufferVariableName()) {
      auto Ty = GV.getValueType();
      auto STy = cast<StructType>(Ty);
      StructTypesNeedingBlock.insert(STy);
    }
  }

  // Traverse the arrays and structures underneath each Block, and
  // mark them as needing layout.
  std::vector<Type *> work_list(StructTypesNeedingBlock.begin(),
                                StructTypesNeedingBlock.end());

  // Physical storage buffer types need layout, but the types can't be found via
  // resource variables. Traverse the module to find the necessary types.
  if (clspv::Option::PhysicalStorageBuffers()) {
    auto add_physical_type = [&](Type *type) {
      if (!type)
        return;
      work_list.push_back(type);
    };

    for (auto &GV : module->globals()) {
      if (GetStorageClass(GV.getAddressSpace()) ==
          spv::StorageClassPhysicalStorageBuffer) {
        add_physical_type(GV.getValueType());
      }
    }

    for (auto &F : *module) {
      for (auto &arg : F.args()) {
        if (arg.getType()->isPointerTy() &&
            GetStorageClass(arg.getType()->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
          auto *inferred_ty =
              clspv::InferType(&arg, module->getContext(), &InferredTypeCache);
          add_physical_type(inferred_ty);
        }
      }
      for (auto &BB : F) {
        for (auto &I : BB) {
          if (auto *gep = dyn_cast<GetElementPtrInst>(&I)) {
            if (GetStorageClass(gep->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              add_physical_type(gep->getSourceElementType());
              add_physical_type(gep->getResultElementType());
            }
          } else if (auto *ld = dyn_cast<LoadInst>(&I)) {
            if (GetStorageClass(ld->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              add_physical_type(ld->getType());
            }
          } else if (auto *st = dyn_cast<StoreInst>(&I)) {
            if (GetStorageClass(st->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              add_physical_type(st->getValueOperand()->getType());
            }
          } else if (auto *atomic = dyn_cast<AtomicRMWInst>(&I)) {
            if (GetStorageClass(atomic->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              add_physical_type(atomic->getValOperand()->getType());
            }
          } else if (auto *cmpxchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
            if (GetStorageClass(cmpxchg->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              add_physical_type(cmpxchg->getNewValOperand()->getType());
            }
          } else if (auto *i2p = dyn_cast<IntToPtrInst>(&I)) {
            if (GetStorageClass(i2p->getType()->getPointerAddressSpace()) ==
                spv::StorageClassPhysicalStorageBuffer) {
              auto *inferred_ty = clspv::InferType(&I, module->getContext(),
                                                   &InferredTypeCache);
              add_physical_type(inferred_ty);
            }
          } else if (auto *cast = dyn_cast<AddrSpaceCastInst>(&I)) {
            if (GetStorageClass(cast->getType()->getPointerAddressSpace()) ==
                    spv::StorageClassPhysicalStorageBuffer ||
                GetStorageClass(cast->getPointerOperand()
                                    ->getType()
                                    ->getPointerAddressSpace()) ==
                    spv::StorageClassPhysicalStorageBuffer) {
              auto *inferred_ty = clspv::InferType(&I, module->getContext(),
                                                   &InferredTypeCache);
              add_physical_type(inferred_ty);
            }
          } else if (auto *call = dyn_cast<CallInst>(&I)) {
            for (auto &arg : call->args()) {
              if (arg->getType()->isPointerTy() &&
                  GetStorageClass(arg->getType()->getPointerAddressSpace()) ==
                      spv::StorageClassPhysicalStorageBuffer) {
                auto *inferred_ty = clspv::InferType(
                    arg.get(), module->getContext(), &InferredTypeCache);
                add_physical_type(inferred_ty);
              }
            }
          }
        }
      }
    }
  }
  while (!work_list.empty()) {
    Type *type = work_list.back();
    work_list.pop_back();
    TypesNeedingLayout.insert(type);
    switch (type->getTypeID()) {
    case Type::ArrayTyID: {
      auto ele_ty = type->getArrayElementType();
      work_list.push_back(ele_ty);
      if (!Hack_generate_runtime_array_stride_early) {
        // Remember this array type for deferred decoration.
        getTypesNeedingArrayStride().insert(
            StrideType(GetTypeAllocSize(ele_ty, module->getDataLayout()),
                       getSPIRVType(type, true)));
      }
      break;
    }
    case Type::StructTyID:
      for (auto *elem_ty : cast<StructType>(type)->elements()) {
        work_list.push_back(elem_ty);
      }
      break;
    default:
      // This type and its contained types don't get layout.
      break;
    }
  }
}

void SPIRVProducerPassImpl::GenerateWorkgroupVars() {
  // Find the local variable resource calls.
  // Clspv only shares spec ids if the data types match so we only need to find
  // one call per spec id.
  DenseMap<int, CallInst *> local_vars;
  int max_spec_id = 0;
  for (auto &F : *module) {
    for (auto &BB : F) {
      for (auto &I : BB) {
        if (auto *call = dyn_cast<CallInst>(&I)) {
          if (Builtins::Lookup(call->getCalledFunction()) ==
              Builtins::kClspvLocal) {
            int spec_id = static_cast<int>(
                cast<ConstantInt>(
                    call->getOperand(ClspvOperand::kWorkgroupSpecId))
                    ->getSExtValue());
            if (!local_vars.count(spec_id)) {
              local_vars[spec_id] = call;
              max_spec_id = std::max(max_spec_id, spec_id);
            }
          }
        }
      }
    }
  }

  // Generate the variables in order of spec id.
  for (int spec_id = 0; spec_id <= max_spec_id; ++spec_id) {
    if (!local_vars.count(spec_id)) {
      continue;
    }

    auto *call = local_vars[spec_id];

    Type *ElemTy = nullptr;
    SPIRVID ArrayTypeID;
    SPIRVID PtrTypeID;
    SPIRVID VariableID;

    // Generate the spec constant.
    SPIRVOperandVec Ops;
    Ops << Type::getInt32Ty(module->getContext()) << 1;
    SPIRVID ArraySizeID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    if (Option::UntypedPointerAddressSpace(AddressSpace::Local)) {
      auto *StructTy =
          call->getArgOperand(ClspvOperand::kWorkgroupDataType)->getType();
      auto *ArrayTy = cast<StructType>(StructTy)->getElementType(0);
      ElemTy = cast<ArrayType>(ArrayTy)->getElementType();

      // Generate the array type.
      Ops.clear();
      Ops << ElemTy << ArraySizeID;
      ArrayTypeID = addSPIRVInst<kTypes>(spv::OpTypeArray, Ops);

      getTypesNeedingArrayStride().insert(
          StrideType(static_cast<uint32_t>(
                         GetTypeAllocSize(ElemTy, module->getDataLayout())),
                     ArrayTypeID));

      // Generate the struct type.
      Ops.clear();
      Ops << ArrayTypeID;
      SPIRVID StructTypeID = addSPIRVInst<kTypes>(spv::OpTypeStruct, Ops);

      // Generate Block and Offset decorations for the struct.
      Ops.clear();
      Ops << StructTypeID << spv::DecorationBlock;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

      Ops.clear();
      Ops << StructTypeID << 0 << spv::DecorationOffset << 0;
      addSPIRVInst<kAnnotations>(spv::OpMemberDecorate, Ops);

      PtrTypeID = getSPIRVPointerType(
          PointerType::get(call->getContext(), AddressSpace::Local), nullptr);

      // Generate the variable.
      // TODO: should arguments be decorated with Aliased? This should be based
      // on the input kernel, but we don't do this for any resource currently.
      Ops.clear();
      Ops << PtrTypeID << spv::StorageClassWorkgroup << StructTypeID;
      VariableID = addSPIRVGlobalVariable(PtrTypeID, spv::StorageClassWorkgroup,
                                          StructTypeID,
                                          /* InitID = */ SPIRVID(0),
                                          /* add_interface = */ false);
    } else {
      auto *ArrayTy =
          call->getArgOperand(ClspvOperand::kWorkgroupDataType)->getType();
      ElemTy = cast<ArrayType>(ArrayTy)->getElementType();

      // Generate the array type.
      Ops.clear();
      // The element type must have been created.
      Ops << ElemTy << ArraySizeID;

      ArrayTypeID = addSPIRVInst<kTypes>(spv::OpTypeArray, Ops);

      Ops.clear();
      Ops << spv::StorageClassWorkgroup << ArrayTypeID;
      PtrTypeID = addSPIRVInst<kTypes>(spv::OpTypePointer, Ops);

      // Generate OpVariable.
      //
      // Ops[0] : Result Type ID
      // Ops[1] : Storage Class
      VariableID = addSPIRVGlobalVariable(PtrTypeID, spv::StorageClassWorkgroup,
                                          /* DataTypeID = */ SPIRVID(0),
                                          /* InitID = */ SPIRVID(0),
                                          /* add_interface = */ false);
    }

    Ops.clear();
    Ops << ArraySizeID << spv::DecorationSpecId << spec_id;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    LocalArgInfo info{VariableID,  ElemTy,    ArraySizeID,
                      ArrayTypeID, PtrTypeID, spec_id};
    LocalSpecIdInfoMap[spec_id] = info;
  }

  // Record the spec ids for each kernel argument. The arguments can be found in
  // the metadata.
  NamedMDNode *nmd = module->getNamedMetadata(clspv::LocalSpecIdMetadataName());
  if (!nmd)
    return;
  for (auto operand : nmd->operands()) {
    MDTuple *tuple = cast<MDTuple>(operand);
    ValueAsMetadata *fn_md = cast<ValueAsMetadata>(tuple->getOperand(0));
    Function *func = cast<Function>(fn_md->getValue());
    ConstantAsMetadata *arg_index_md =
        cast<ConstantAsMetadata>(tuple->getOperand(1));
    int arg_index = static_cast<int>(
        cast<ConstantInt>(arg_index_md->getValue())->getSExtValue());
    Argument *arg = &*(func->arg_begin() + arg_index);

    ConstantAsMetadata *spec_id_md =
        cast<ConstantAsMetadata>(tuple->getOperand(2));
    int spec_id = static_cast<int>(
        cast<ConstantInt>(spec_id_md->getValue())->getSExtValue());

    LocalArgSpecIds[arg] = spec_id;
  }
}

spv::StorageClass SPIRVProducerPassImpl::GetStorageBufferClass() const {
  return clspv::Option::PhysicalStorageBuffers()
             ? spv::StorageClassPhysicalStorageBuffer
             : spv::StorageClassStorageBuffer;
}

spv::StorageClass
SPIRVProducerPassImpl::GetStorageClass(unsigned AddrSpace) const {
  switch (AddrSpace) {
  case AddressSpace::Private:
    return spv::StorageClassFunction;
  case AddressSpace::Global:
    return GetStorageBufferClass();
  case AddressSpace::Constant:
    return clspv::Option::ConstantArgsInUniformBuffer()
               ? spv::StorageClassUniform
               : GetStorageBufferClass();
  case AddressSpace::Input:
    return spv::StorageClassInput;
  case AddressSpace::Local:
    return spv::StorageClassWorkgroup;
  case AddressSpace::UniformConstant:
    return spv::StorageClassUniformConstant;
  case AddressSpace::Uniform:
    return spv::StorageClassUniform;
  case AddressSpace::ModuleScopePrivate:
    return spv::StorageClassPrivate;
  case AddressSpace::PushConstant:
    return spv::StorageClassPushConstant;
  default:
    llvm_unreachable("Unsupported OpenCL address space");
  }
}

spv::StorageClass SPIRVProducerPassImpl::GetStorageClassForArgKind(
    clspv::ArgKind arg_kind) const {
  switch (arg_kind) {
  case clspv::ArgKind::Buffer:
    return spv::StorageClassStorageBuffer;
  case clspv::ArgKind::BufferUBO:
    return spv::StorageClassUniform;
  case clspv::ArgKind::Pod:
    return spv::StorageClassStorageBuffer;
  case clspv::ArgKind::PodUBO:
  case clspv::ArgKind::PointerUBO:
    return spv::StorageClassUniform;
  case clspv::ArgKind::PodPushConstant:
    return spv::StorageClassPushConstant;
  case clspv::ArgKind::Local:
    return spv::StorageClassWorkgroup;
  case clspv::ArgKind::SampledImage:
  case clspv::ArgKind::StorageImage:
  case clspv::ArgKind::Sampler:
  case clspv::ArgKind::StorageTexelBuffer:
  case clspv::ArgKind::UniformTexelBuffer:
    return spv::StorageClassUniformConstant;
  default:
    llvm_unreachable("Unsupported storage class for argument kind");
  }
}

spv::BuiltIn SPIRVProducerPassImpl::GetBuiltin(StringRef Name) const {
  return StringSwitch<spv::BuiltIn>(Name)
      .Case("__spirv_GlobalInvocationId", spv::BuiltInGlobalInvocationId)
      .Case("__spirv_LocalInvocationId", spv::BuiltInLocalInvocationId)
      .Case("__spirv_WorkgroupSize", spv::BuiltInWorkgroupSize)
      .Case("__spirv_NumWorkgroups", spv::BuiltInNumWorkgroups)
      .Case("__spirv_WorkgroupId", spv::BuiltInWorkgroupId)
      .Case("__spirv_WorkDim", spv::BuiltInWorkDim)
      .Case("__spirv_GlobalOffset", spv::BuiltInGlobalOffset)
      .Case("__spirv_SubgroupMaxSize", spv::BuiltInSubgroupMaxSize)
      .Default(spv::BuiltInMax);
}

SPIRVID SPIRVProducerPassImpl::getOpExtInstImportID() {
  if (OpExtInstImportID == 0) {
    //
    // Generate OpExtInstImport.
    //
    // Ops[0] ... Ops[n] = Name (Literal String)

    OpExtInstImportID =
        addSPIRVInst<kImports>(spv::OpExtInstImport, "GLSL.std.450");
  }
  return OpExtInstImportID;
}

SPIRVID SPIRVProducerPassImpl::addSPIRVGlobalVariable(const SPIRVID &TypeID,
                                                      spv::StorageClass SC,
                                                      const SPIRVID &DataTypeID,
                                                      const SPIRVID &InitID,
                                                      bool add_interface) {
  // Generate OpVariable.
  //
  // Ops[0] : Result Type ID
  // Ops[1] : Storage Class
  // Ops[2] : Initialization Value ID (optional)

  const bool untyped = UntypedPointerStorageClass(SC);
  SPIRVOperandVec Ops;
  Ops << TypeID << SC;
  if (untyped) {
    Ops << DataTypeID;
  }
  if (InitID.isValid()) {
    Ops << InitID;
  }

  SPIRVID VID;
  if (untyped) {
    VID = addSPIRVInst<kGlobalVariables>(spv::OpUntypedVariableKHR, Ops);
    if (SC == spv::StorageClassWorkgroup) {
      addCapability(spv::CapabilityWorkgroupMemoryExplicitLayoutKHR);
    }
  } else {
    VID = addSPIRVInst<kGlobalVariables>(spv::OpVariable, Ops);
  }

  if (SC == spv::StorageClassInput ||
      (add_interface && SpvVersion() >= SPIRVVersion::SPIRV_1_4)) {
    getEntryPointInterfacesList().push_back(VID);
  }

  return VID;
}

SPIRVID SPIRVProducerPassImpl::ChangeLayout(SPIRVID object, Type *type,
                                            bool removeLayout) {
  auto dst_id = getSPIRVType(type, !removeLayout);

  SPIRVID copy = object;
  if (Option::SpvVersion() >= SPIRVVersion::SPIRV_1_4) {
    SPIRVOperandVec Ops;
    Ops << dst_id << object;
    copy = addSPIRVInst(spv::OpCopyLogical, Ops);
  } else {
    // Recursively perform a manual OpCopyLogical.
    auto struct_ty = dyn_cast<StructType>(type);
    auto array_ty = dyn_cast<ArrayType>(type);
    if (struct_ty || array_ty) {
      SPIRVOperandVec constructOps;
      constructOps << dst_id;
      uint32_t num_eles =
          struct_ty ? struct_ty->getNumElements() : array_ty->getNumElements();
      for (uint32_t i = 0; i < num_eles; i++) {
        auto sub_type = struct_ty ? struct_ty->getElementType(i)
                                  : array_ty->getElementType();

        // Extract from src
        SPIRVOperandVec Ops;
        auto type_id = getSPIRVType(sub_type, removeLayout);
        Ops << type_id << object << i;
        auto extract = addSPIRVInst(spv::OpCompositeExtract, Ops);

        // Swap layout
        auto swapped = ChangeLayout(extract, sub_type, removeLayout);

        // Add to the composite construct
        constructOps << swapped;
      }

      copy = addSPIRVInst(spv::OpCompositeConstruct, constructOps);
    }
  }

  return copy;
}

Type *SPIRVProducerPassImpl::CanonicalType(Type *type) {
  if (type->isIntegerTy()) {
    auto bit_width = static_cast<uint32_t>(type->getPrimitiveSizeInBits());
    if (bit_width > 1) {
      // round up bit_width to a multiple of 8
      bit_width = ((bit_width + 7) / 8) * 8;
    }
    return IntegerType::get(type->getContext(), bit_width);
  } else if (auto *ptr_ty = dyn_cast<PointerType>(type)) {
    // For the purposes of our Vulkan SPIR-V type system, constant and global
    // are conflated.
    unsigned AddrSpace = ptr_ty->getAddressSpace();
    if (AddressSpace::Constant == AddrSpace) {
      if (!clspv::Option::ConstantArgsInUniformBuffer() &&
          !clspv::Option::PhysicalStorageBuffers()) {
        AddrSpace = AddressSpace::Global;
        // The canonical type of __constant is __global unless constants are
        // passed in uniform buffers.
        auto *GlobalTy = PointerType::get(type->getContext(), AddrSpace);
        return GlobalTy;
      }
    }
  } else if (type->isTargetExtTy()) {
    // Nothing
  } else if (type->getNumContainedTypes() != 0) {
    switch (type->getTypeID()) {
    case Type::StructTyID: {
      SmallVector<Type *, 8> subtypes;
      bool changed = false;
      for (auto *subtype : type->subtypes()) {
        auto canonical = CanonicalType(subtype);
        subtypes.push_back(canonical);
        if (canonical != subtype) {
          changed = true;
        }
      }
      if (changed) {
        return StructType::get(type->getContext(), subtypes,
                               cast<StructType>(type)->isPacked());
      }
      break;
    }
    case Type::ArrayTyID: {
      auto *elem_ty = type->getArrayElementType();
      auto *equiv_elem_ty = CanonicalType(elem_ty);
      if (equiv_elem_ty != elem_ty) {
        return ArrayType::get(equiv_elem_ty,
                              cast<ArrayType>(type)->getNumElements());
      }
      break;
    }
    case Type::FunctionTyID: {
      auto *func_ty = cast<FunctionType>(type);
      auto *return_ty = CanonicalType(func_ty->getReturnType());
      SmallVector<Type *, 8> params;
      for (unsigned i = 0; i < func_ty->getNumParams(); ++i) {
        params.push_back(CanonicalType(func_ty->getParamType(i)));
      }
      return FunctionType::get(return_ty, params, func_ty->isVarArg());
    }
    default:
      break;
    }
  }

  return type;
}

bool SPIRVProducerPassImpl::PointerRequiresLayout(unsigned aspace) {
  switch (aspace) {
  case AddressSpace::PushConstant:
  case AddressSpace::Uniform:
  case AddressSpace::Global:
  case AddressSpace::Constant:
    return true;
  case AddressSpace::Local:
    return Option::UntypedPointerAddressSpace(aspace);
  default:
    break;
  }
  return false;
}

bool SPIRVProducerPassImpl::UntypedPointerStorageClass(spv::StorageClass sc) {
  if (!Option::UntypedPointers()) {
    return false;
  }

  switch (sc) {
  case spv::StorageClassStorageBuffer:
  case spv::StorageClassUniform:
  case spv::StorageClassPushConstant:
  case spv::StorageClassPhysicalStorageBuffer:
    return true;
  case spv::StorageClassWorkgroup:
    return Option::SpvVersion() >= Option::SPIRVVersion::SPIRV_1_4;
  default:
    break;
  }

  return false;
}

SPIRVID SPIRVProducerPassImpl::getSPIRVPointerType(Type *PtrTy, Type *DataTy) {
  if (!PtrTy->isPointerTy()) {
    return getSPIRVType(PtrTy);
  }

  const auto aspace = PtrTy->getPointerAddressSpace();
  auto canonical_aspace = aspace;
  if (aspace == AddressSpace::Constant &&
      !Option::ConstantArgsInUniformBuffer()) {
    canonical_aspace = AddressSpace::Global;
  }

  if (Option::UntypedPointerAddressSpace(canonical_aspace)) {
    auto &aspace_map = PointerTypeMap[canonical_aspace];
    auto where = aspace_map.find(PtrTy);
    if (where != aspace_map.end()) {
      if (where->second[1].isValid()) {
        return where->second[1];
      }
    }

    addCapability(spv::CapabilityUntypedPointersKHR);
    SPIRVOperandVec ops;
    ops << GetStorageClass(canonical_aspace);
    auto ptr_id = addSPIRVInst<kTypes>(spv::OpTypeUntypedPointerKHR, ops);

    auto &entry = aspace_map[PtrTy];
    if (entry.empty())
      entry.resize(2);
    entry[1] = ptr_id;

    return ptr_id;
  }

  if (!DataTy) {
    llvm_unreachable("Missing inferred type");
  }
  const auto needs_layout = PointerRequiresLayout(aspace);

  // This might still be problematic for structures/arrays of pointers. Clspv
  // can only support those in private and function storage classes though.
  auto data_id = getSPIRVType(DataTy, needs_layout);

  // Images and samplers can only be in the UniformConstant address space, so
  // clspv uses them by value in most parts of the code generation. The initial
  // generation of the resource variable and uses generate the correct pointer
  // and load of the type.
  if (auto struct_ty = dyn_cast<StructType>(DataTy)) {
    if (struct_ty->isOpaque() && aspace != AddressSpace::UniformConstant) {
      return data_id;
    }
  }

  const unsigned layout_index = needs_layout ? 1 : 0;
  auto &aspace_map = PointerTypeMap[canonical_aspace];
  auto where = aspace_map.find(DataTy);
  if (where != aspace_map.end()) {
    if (where->second[layout_index].isValid()) {
      return where->second[layout_index];
    }
  }

  // Generate the SPIR-V pointer type.
  SPIRVOperandVec ops;
  ops << GetStorageClass(canonical_aspace) << data_id;
  auto ptr_id = addSPIRVInst<kTypes>(spv::OpTypePointer, ops);

  auto &entry = aspace_map[DataTy];
  if (entry.empty())
    entry.resize(2);
  entry[layout_index] = ptr_id;

  return ptr_id;
}

SPIRVID SPIRVProducerPassImpl::getSPIRVFunctionType(FunctionType *FTy,
                                                    Type *RetTy,
                                                    ArrayRef<Type *> ParamTys) {
  bool has_pointers = false;
  if (FTy->getReturnType()->isPointerTy()) {
    has_pointers = true;
  }
  for (auto *param_ty : FTy->params()) {
    if (param_ty->isPointerTy()) {
      has_pointers = true;
      break;
    }
  }

  // If there are no opaque pointer parameters defer to basic type handler.
  if (!has_pointers) {
    return getSPIRVType(FTy);
  }

  // Create a placeholder function type for use in caching. Pointer parameters
  // and return are mangled as placeholder parameter types to avoid collisions.
  // We use arrays for this purpose: [aspace x inferred_ty].
  SmallVector<Type *, 8> placeholder_param_tys;
  int i = 0;
  for (auto *param_ty : FTy->params()) {
    param_ty = CanonicalType(param_ty);
    placeholder_param_tys.push_back(param_ty);
    if (param_ty->isPointerTy()) {
      // Generate a representative type for the parameter type.
      unsigned aspace = param_ty->getPointerAddressSpace();
      if (Option::UntypedPointerAddressSpace(aspace)) {
        auto *inferred_ty = ParamTys[i];
        auto *rep_ty = ArrayType::get(inferred_ty, aspace);
        placeholder_param_tys.push_back(rep_ty);
      }
    }
    i++;
  }
  if (auto ptr_ty = dyn_cast<PointerType>(FTy->getReturnType())) {
    // Generate a representative type for the return type.
    auto *ty = CanonicalType(ptr_ty);
    unsigned aspace = ty->getPointerAddressSpace();
    if (Option::UntypedPointerAddressSpace(aspace)) {
      auto *rep_ty = ArrayType::get(RetTy, aspace);
      placeholder_param_tys.push_back(rep_ty);
    }
  }

  auto *placeholder_ty =
      FunctionType::get(CanonicalType(FTy->getReturnType()),
                        placeholder_param_tys, FTy->isVarArg());
  auto where = FunctionTypeMap.find(placeholder_ty);
  if (where != FunctionTypeMap.end()) {
    return where->second;
  }

  // Generate OpTypeFunction.
  // [0] = Result type
  // [1..n] = Param types
  SPIRVOperandVec ops;
  ops << getSPIRVPointerType(FTy->getReturnType(), RetTy);
  i = 0;
  for (auto *param_ty : FTy->params()) {
    auto *inferred_ty = ParamTys[i++];
    ops << getSPIRVPointerType(param_ty, inferred_ty);
  }
  auto id = addSPIRVInst<kTypes>(spv::OpTypeFunction, ops);
  FunctionTypeMap[placeholder_ty] = id;
  return id;
}

SPIRVID SPIRVProducerPassImpl::getSPIRVType(Type *Ty) {
  // Prior to 1.4, layout decorations are more relaxed so we can reuse a laid
  // out type in non-laid out storage classes.
  bool needs_layout = false;
  if (auto ptr_ty = dyn_cast<PointerType>(Ty)) {
    needs_layout = PointerRequiresLayout(ptr_ty->getPointerAddressSpace());
  }
  return getSPIRVType(Ty, needs_layout);
}

SPIRVID SPIRVProducerPassImpl::getSPIRVType(Type *Ty, bool needs_layout) {
  if (Ty->isPointerTy()) {
    llvm_unreachable("Unsupported opaque pointer");
  }

  // Only pointers, non-opaque structs and arrays should have layout
  // decorations.
  if (!(isa<PointerType>(Ty) || isa<ArrayType>(Ty) || isa<StructType>(Ty))) {
    needs_layout = false;
  } else if (auto StructTy = dyn_cast<StructType>(Ty)) {
    if (StructTy->isOpaque()) {
      needs_layout = false;
    }
  }
  // |layout| is the index used for |Ty|'s entry in the type map. Each type
  // stores a laid out and non-laid out version of the type.
  const unsigned layout = needs_layout ? 1 : 0;

  auto TI = TypeMap.find(Ty);
  if (TI != TypeMap.end()) {
    assert(layout < TI->second.size());
    if (TI->second[layout].isValid()) {
      return TI->second[layout];
    }
  }

  auto Canonical = CanonicalType(Ty);

  if (Canonical != Ty) {
    auto CanonicalTI = TypeMap.find(Canonical);
    if (CanonicalTI != TypeMap.end()) {
      assert(layout < CanonicalTI->second.size());
      if (CanonicalTI->second[layout].isValid()) {
        auto id = CanonicalTI->second[layout];
        auto &base = TypeMap[Ty];
        if (base.empty()) {
          base.resize(2);
        }
        base[layout] = id;
        return id;
      }
    }
  }

  // Perform the mapping with the canonical type.

  const auto &DL = module->getDataLayout();

  SPIRVID RID;

  switch (Canonical->getTypeID()) {
  default: {
    Canonical->print(errs());
    llvm_unreachable("Unsupported type???");
    break;
  }
  case Type::MetadataTyID:
  case Type::LabelTyID: {
    // Ignore these types.
    break;
  }
  case Type::PointerTyID: {
    llvm_unreachable("getSPIRVType called directly on a pointer type");
    break;
  }
  case Type::TargetExtTyID: {
    auto *ext_ty = cast<TargetExtType>(Canonical);
    if (IsImageType(ext_ty)) {
      const auto dim = ImageDimensionality(ext_ty);
      const auto sampled = IsSampledImageType(ext_ty);
      switch (dim) {
        case spv::Dim1D:
          if (sampled) {
            addCapability(spv::CapabilitySampled1D);
          } else {
            addCapability(spv::CapabilityImage1D);
          }
          break;
        case spv::DimBuffer:
          if (sampled) {
            addCapability(spv::CapabilitySampledBuffer);
          } else {
            addCapability(spv::CapabilityImageBuffer);
          }
          break;
        default:
          break;
      }

      //
      // Generate OpTypeImage
      //
      // Ops[0] = Sampled Type ID
      // Ops[1] = Dim ID
      // Ops[2] = Depth (Literal Number)
      // Ops[3] = Arrayed (Literal Number)
      // Ops[4] = MS (Literal Number)
      // Ops[5] = Sampled (Literal Number)
      // Ops[6] = Image Format ID
      //
      SPIRVOperandVec Ops;

      SPIRVID SampledTyID;
      // None of the sampled types have a layout.
      if (IsFloatImageType(ext_ty)) {
          SampledTyID =
              getSPIRVType(Type::getFloatTy(Canonical->getContext()), false);
      } else if (IsUintImageType(ext_ty)) {
          SampledTyID =
              getSPIRVType(Type::getInt32Ty(Canonical->getContext()), false);
      } else if (IsIntImageType(ext_ty)) {
          // Generate a signed 32-bit integer if necessary.
          if (int32ID == 0) {
            SPIRVOperandVec intOps;
            intOps << 32 << 1;
            int32ID = addSPIRVInst<kTypes>(spv::OpTypeInt, intOps);
          }
          SampledTyID = int32ID;

          // Generate a vec4 of the signed int if necessary.
          if (v4int32ID == 0) {
            SPIRVOperandVec vecOps;
            vecOps << int32ID << 4;
            v4int32ID = addSPIRVInst<kTypes>(spv::OpTypeVector, vecOps);
          }
      } else {
          // This was likely an UndefValue.
          SampledTyID =
              getSPIRVType(Type::getFloatTy(Canonical->getContext()), false);
      }
      Ops << SampledTyID;

      spv::Dim DimID = ImageDimensionality(ext_ty);
      Ops << DimID;

      // TODO: Set up Depth.
      Ops << 0;

      uint32_t arrayed = IsArrayImageType(ext_ty) ? 1 : 0;
      Ops << arrayed;

      // TODO: Set up MS.
      Ops << 0;

      // Set up Sampled.
      //
      // From Spec
      //
      // 0 indicates this is only known at run time, not at compile time
      // 1 indicates will be used with sampler
      // 2 indicates will be used without a sampler (a storage image)
      uint32_t Sampled = 1;
      if (!IsSampledImageType(ext_ty)) {
          Sampled = 2;
      }
      Ops << Sampled;

      // TODO: Set up Image Format.
      Ops << spv::ImageFormatUnknown;
      RID = addSPIRVInst<kTypes>(spv::OpTypeImage, Ops);

      // Only need a sampled version of the type if it is used with a sampler.
      // In SPIR-V 1.6 or later, sampled image dimension must not be Buffer
      if (Sampled == 1 && ImageDimensionality(ext_ty) != spv::DimBuffer) {
          Ops.clear();
          Ops << RID;
          getImageTypeMap()[Canonical] =
              addSPIRVInst<kTypes>(spv::OpTypeSampledImage, Ops);
      }
      break;
    } else if (IsSamplerType(ext_ty)) {
      //
      // Generate OpTypeSampler
      //
      // Empty Ops.

      RID = addSPIRVInst<kTypes>(spv::OpTypeSampler);
      break;
    } else {
      llvm_unreachable("Unknown target ext type");
    }
    break;
  }
  case Type::StructTyID: {
    StructType *STy = cast<StructType>(Canonical);

    if (STy->getNumElements() == 1 && STy->getElementType(0)->isArrayTy() &&
        cast<ArrayType>(STy->getElementType(0))->getNumElements() == 0) {
      StructTypesNeedingBlock.insert(STy);
    }

    //
    // Generate OpTypeStruct
    //
    // Ops[0] ... Ops[n] = Member IDs
    SPIRVOperandVec Ops;

    for (auto *EleTy : STy->elements()) {
      Ops << getSPIRVType(EleTy, needs_layout);
    }

    RID = addSPIRVInst<kTypes>(spv::OpTypeStruct, Ops);

    // Generate OpMemberDecorate unless we are generating it for the canonical
    // type.
    StructType *canonical = cast<StructType>(CanonicalType(STy));
    if (TypesNeedingLayout.idFor(STy) &&
        (canonical == STy || !TypesNeedingLayout.idFor(canonical)) &&
        needs_layout) {
      for (unsigned MemberIdx = 0; MemberIdx < STy->getNumElements();
           MemberIdx++) {
        // Ops[0] = Structure Type ID
        // Ops[1] = Member Index(Literal Number)
        // Ops[2] = Decoration (Offset)
        // Ops[3] = Byte Offset (Literal Number)
        const auto ByteOffset =
            GetExplicitLayoutStructMemberOffset(STy, MemberIdx, DL);

        Ops.clear();
        Ops << RID << MemberIdx << spv::DecorationOffset << ByteOffset;

        addSPIRVInst<kAnnotations>(spv::OpMemberDecorate, Ops);
      }
    }

    // Generate OpDecorate unless we are generating it for the canonical type.
    if (StructTypesNeedingBlock.idFor(STy) &&
        (canonical == STy || !StructTypesNeedingBlock.idFor(canonical)) &&
        needs_layout) {
      Ops.clear();
      // Use Block decorations with StorageBuffer storage class.
      Ops << RID << spv::DecorationBlock;

      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
    }
    break;
  }
  case Type::IntegerTyID: {
    uint32_t bit_width =
        static_cast<uint32_t>(Canonical->getPrimitiveSizeInBits());

    if (clspv::Option::Int8Support() && bit_width == 8) {
      addCapability(spv::CapabilityInt8);
    } else if (bit_width == 16) {
      addCapability(spv::CapabilityInt16);
    } else if (bit_width == 64) {
      addCapability(spv::CapabilityInt64);
    }

    if (bit_width == 1) {
      RID = addSPIRVInst<kTypes>(spv::OpTypeBool);
    } else {
      if (!clspv::Option::Int8Support() && bit_width == 8) {
        // i8 is added to TypeMap as i32.
        RID = getSPIRVType(Type::getIntNTy(Canonical->getContext(), 32), false);
      } else {
        SPIRVOperandVec Ops;
        Ops << bit_width << 0 /* not signed */;
        RID = addSPIRVInst<kTypes>(spv::OpTypeInt, Ops);
      }
    }
    break;
  }
  case Type::HalfTyID:
  case Type::FloatTyID:
  case Type::DoubleTyID: {
    uint32_t bit_width =
        static_cast<uint32_t>(Canonical->getPrimitiveSizeInBits());
    if (bit_width == 16) {
      addCapability(spv::CapabilityFloat16);
    } else if (bit_width == 64) {
      addCapability(spv::CapabilityFloat64);
    }

    SPIRVOperandVec Ops;
    Ops << bit_width;

    RID = addSPIRVInst<kTypes>(spv::OpTypeFloat, Ops);
    break;
  }
  case Type::ArrayTyID: {
    ArrayType *ArrTy = cast<ArrayType>(Canonical);
    const uint64_t Length = ArrTy->getArrayNumElements();
    if (Length == 0) {
      // By convention, map it to a RuntimeArray.

      Type *EleTy = ArrTy->getArrayElementType();

      //
      // Generate OpTypeRuntimeArray.
      //
      // OpTypeRuntimeArray
      // Ops[0] = Element Type ID
      SPIRVOperandVec Ops;
      Ops << getSPIRVType(EleTy, needs_layout);

      RID = addSPIRVInst<kTypes>(spv::OpTypeRuntimeArray, Ops);

      if (Hack_generate_runtime_array_stride_early && needs_layout) {
        // Generate OpDecorate.

        // Ops[0] = Target ID
        // Ops[1] = Decoration (ArrayStride)
        // Ops[2] = Stride Number(Literal Number)
        Ops.clear();

        auto CurrStride = static_cast<uint32_t>(GetTypeAllocSize(EleTy, DL));
        // if stride is 1 automatically set it to 4 to avoid stride issues.
        Ops << RID << spv::DecorationArrayStride
            << getCorrectedStride(CurrStride);

        addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
      }

    } else {

      //
      // Generate OpConstant and OpTypeArray.
      //

      //
      // Generate OpConstant for array length.
      //
      // Add constant for length to constant list.
      Constant *CstLength =
          ConstantInt::get(Type::getInt32Ty(module->getContext()), Length);

      //
      // Generate OpTypeArray.
      //
      // Ops[0] = Element Type ID
      // Ops[1] = Array Length Constant ID
      SPIRVOperandVec Ops;

      Ops << getSPIRVType(ArrTy->getElementType(), needs_layout) << CstLength;

      RID = addSPIRVInst<kTypes>(spv::OpTypeArray, Ops);

      // Remember to generate ArrayStride later
      if (needs_layout) {
        getTypesNeedingArrayStride().insert(StrideType(
            GetTypeAllocSize(ArrTy->getElementType(), module->getDataLayout()),
            RID));
      }
    }
    break;
  }
  case Type::FixedVectorTyID: {
    auto VecTy = cast<VectorType>(Canonical);
    // <4 x i8> is changed to i32 if i8 is not generally supported.
    if (!clspv::Option::Int8Support() &&
        VecTy->getElementType() == Type::getInt8Ty(module->getContext())) {
      if (VecTy->getElementCount().getKnownMinValue() == 4) {
        RID = getSPIRVType(VecTy->getElementType());
        break;
      } else {
        Canonical->print(errs());
        llvm_unreachable("Support above i8 vector type");
      }
    }

    if (VecTy->getElementCount().getKnownMinValue() == 1) {
      RID = getSPIRVType(VecTy->getElementType());
      break;
    }

    // Ops[0] = Component Type ID
    // Ops[1] = Component Count (Literal Number)
    SPIRVOperandVec Ops;
    Ops << VecTy->getElementType()
        << VecTy->getElementCount().getKnownMinValue();

    RID = addSPIRVInst<kTypes>(spv::OpTypeVector, Ops);
    break;
  }
  case Type::VoidTyID: {
    RID = addSPIRVInst<kTypes>(spv::OpTypeVoid);
    break;
  }
  case Type::FunctionTyID: {
    // Generate SPIRV instruction for function type.
    FunctionType *FTy = cast<FunctionType>(Canonical);

    // Ops[0] = Return Type ID
    // Ops[1] ... Ops[n] = Parameter Type IDs
    SPIRVOperandVec Ops;

    // Find SPIRV instruction for return type
    Ops << FTy->getReturnType();

    // Find SPIRV instructions for parameter types
    for (unsigned k = 0; k < FTy->getNumParams(); k++) {
      // Find SPIRV instruction for parameter type.
      auto ParamTy = FTy->getParamType(k);
      Ops << getSPIRVType(ParamTy);
    }

    RID = addSPIRVInst<kTypes>(spv::OpTypeFunction, Ops);
    break;
  }
  }

  if (RID.isValid()) {
    auto &entry = TypeMap[Canonical];
    if (entry.empty()) {
      entry.resize(2);
    }
    entry[layout] = RID;

    if (Canonical != Ty) {
      // Also cache the original type.
      auto &base_entry = TypeMap[Ty];
      if (base_entry.empty()) {
        base_entry.resize(2);
      }
      base_entry[layout] = RID;
    }
  }
  return RID;
}

SPIRVID SPIRVProducerPassImpl::getSPIRVInt32Constant(uint32_t CstVal) {
  Type *i32 = Type::getInt32Ty(module->getContext());
  Constant *Cst = ConstantInt::get(i32, CstVal);
  return getSPIRVValue(Cst);
}

SPIRVID SPIRVProducerPassImpl::getSPIRVConstant(Constant *C, Type *TyHint) {
  ValueMapType &VMap = getValueMap();
  const bool hack_undef = clspv::Option::HackUndef();

  // Treat poison as an undef.
  auto *Cst = C;
  if (isa<PoisonValue>(Cst)) {
    Cst = UndefValue::get(Cst->getType());
  }

  auto VI = VMap.find(Cst);
  if (VI != VMap.end()) {
    assert(VI->second.isValid());
    return VI->second;
  }

  SPIRVID RID;

  //
  // Generate OpConstant.
  //
  // Ops[0] = Result Type ID
  // Ops[1] .. Ops[n] = Values LiteralNumber
  SPIRVOperandVec Ops;

  if (Cst->getType()->isPointerTy()) {
    auto *inferred_ty =
        clspv::InferType(Cst, Cst->getContext(), &InferredTypeCache, TyHint);
    Ops << getSPIRVPointerType(Cst->getType(), inferred_ty);
  } else {
    Ops << Cst->getType();
  }

  std::vector<uint32_t> LiteralNum;
  spv::Op Opcode = spv::OpNop;

  if (isa<UndefValue>(Cst)) {
    // Ops[0] = Result Type ID
    Opcode = spv::OpUndef;
    if (hack_undef && IsTypeNullable(Cst->getType())) {
      Opcode = spv::OpConstantNull;
    }
  } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(Cst)) {
    unsigned bit_width = CI->getBitWidth();
    if (bit_width == 1) {
      // If the bitwidth of constant is 1, generate OpConstantTrue or
      // OpConstantFalse.
      if (CI->getZExtValue()) {
        // Ops[0] = Result Type ID
        Opcode = spv::OpConstantTrue;
      } else {
        // Ops[0] = Result Type ID
        Opcode = spv::OpConstantFalse;
      }
    } else {
      auto V = CI->getZExtValue();
      if (bit_width == 16) {
        V &= 0xFFFF;
      }
      LiteralNum.push_back(V & 0xFFFFFFFF);

      if (bit_width > 32) {
        LiteralNum.push_back(V >> 32);
      }

      Opcode = spv::OpConstant;

      Ops << LiteralNum;
    }
  } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Cst)) {
    uint64_t FPVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
    Type *CFPTy = CFP->getType();

    if (auto *vecTy = dyn_cast<llvm::VectorType>(CFPTy)) {
      // A splat over floats is represented as a ConstantFP.
      Opcode = spv::OpConstantComposite;

      Constant *scalar =
          ConstantFP::get(vecTy->getElementType(), CFP->getValueAPF());
      SPIRVID spirv_scalar = getSPIRVConstant(scalar);
      auto num_elem = vecTy->getElementCount().getFixedValue();
      while (num_elem-- > 0) {
        Ops << spirv_scalar;
      }
    } else {
      if (CFPTy->isFloatTy()) {
        LiteralNum.push_back(FPVal & 0xFFFFFFFF);
      } else if (CFPTy->isDoubleTy()) {
        LiteralNum.push_back(FPVal & 0xFFFFFFFF);
        LiteralNum.push_back(FPVal >> 32);
      } else if (CFPTy->isHalfTy()) {
        LiteralNum.push_back(FPVal & 0xFFFF);
      } else {
        CFPTy->print(errs());
        llvm_unreachable("Implement this ConstantFP Type");
      }

      Opcode = spv::OpConstant;
      Ops << LiteralNum;
    }
  } else if (isa<ConstantDataSequential>(Cst) &&
             cast<ConstantDataSequential>(Cst)->isString()) {

    const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(Cst);

    Opcode = spv::OpConstantComposite;
    for (unsigned k = 0; k < CDS->getNumElements(); k++) {
      Ops << CDS->getElementAsConstant(k);
    }
  } else if (const ConstantDataSequential *CDS =
                 dyn_cast<ConstantDataSequential>(Cst)) {
    // Let's convert <4 x i8> constant to int constant specially.
    // This case occurs when all the values are specified as constant
    // ints.
    Type *CstTy = Cst->getType();
    if (is4xi8vec(CstTy)) {
      //
      // Generate OpConstant with OpTypeInt 32 0.
      //
      uint32_t IntValue = 0;
      for (unsigned k = 0; k < 4; k++) {
        const uint64_t Val = CDS->getElementAsInteger(k);
        IntValue = (IntValue << 8) | (Val & 0xffu);
      }

      RID = getSPIRVInt32Constant(IntValue);
    } else if (CDS->getNumElements() > 1) {

      // A normal constant-data-sequential case.
      for (unsigned k = 0; k < CDS->getNumElements(); k++) {
        Ops << CDS->getElementAsConstant(k);
      }

      Opcode = spv::OpConstantComposite;
    } else {
      return getSPIRVValue(CDS->getElementAsConstant(0));
    }
  } else if (const ConstantAggregate *CA = dyn_cast<ConstantAggregate>(Cst)) {
    // Let's convert <4 x i8> constant to int constant specially.
    // This case occurs when at least one of the values is an undef.
    Type *CstTy = Cst->getType();
    if (is4xi8vec(CstTy)) {
      //
      // Generate OpConstant with OpTypeInt 32 0.
      //
      uint32_t IntValue = 0;
      for (User::const_op_iterator I = Cst->op_begin(), E = Cst->op_end();
           I != E; ++I) {
        uint64_t Val = 0;
        const Value *CV = *I;
        if (auto *CI2 = dyn_cast<ConstantInt>(CV)) {
          Val = CI2->getZExtValue();
        }
        IntValue = (IntValue << 8) | (Val & 0xffu);
      }

      RID = getSPIRVInt32Constant(IntValue);
    } else {

      // We use a constant composite in SPIR-V for our constant aggregate in
      // LLVM.
      Opcode = spv::OpConstantComposite;

      for (unsigned k = 0; k < CA->getNumOperands(); k++) {
        // And add an operand to the composite we are constructing
        Ops << CA->getAggregateElement(k);
      }
    }
  } else if (Cst->isNullValue()) {
    Opcode = spv::OpConstantNull;
  } else if(const Function *Fn = dyn_cast<Function>(Cst)) {
    if (Fn->isIntrinsic()) {
      Fn->print(errs());
      llvm_unreachable("Unsupported llvm intrinsic");
    }
    Fn->print(errs());
    llvm_unreachable("Unhandled function declaration/definition");
  } else if (auto *ConstExpr = dyn_cast<ConstantExpr>(Cst)) {
    // If there is exactly one use we know where to insert the instruction
    if (ConstExpr->getNumUses() == 1) {
      auto *User = *ConstExpr->user_begin();
      auto *EquivInstr = ConstExpr->getAsInstruction();
      EquivInstr->insertBefore(dyn_cast<Instruction>(User)->getIterator());
      GenerateInstruction(*EquivInstr);
      RID = VMap[EquivInstr];
    } else {
      llvm_unreachable("Unhandled ConstantExpr");
    }
  } else {
    Cst->print(errs());
    llvm_unreachable("Unsupported Constant???");
  }

  if (Opcode == spv::OpConstantNull && Cst->getType()->isPointerTy()) {
    // Null pointer requires variable pointers.
    setVariablePointersCapabilities(Cst->getType()->getPointerAddressSpace());
  }

  if (RID == 0) {
    RID = addSPIRVInst<kConstants>(Opcode, Ops);
  }

  VMap[Cst] = RID;

  return RID;
}

SPIRVID SPIRVProducerPassImpl::getSPIRVValue(Value *V, Type *TyHint) {
  auto II = ValueMap.find(V);
  if (II != ValueMap.end()) {
    assert(II->second.isValid());
    return II->second;
  }
  if (isa<Function>(V)) {
    return SPIRVID();
  }
  if (Constant *Cst = dyn_cast<Constant>(V)) {
    return getSPIRVConstant(Cst, TyHint);
  } else {
    llvm_unreachable("Variable not found");
  }
}

SPIRVID
SPIRVProducerPassImpl::getSPIRVPointerOperand(Value *PtrVal,
                                              Type *ExpectedPointeeTy) {
  SPIRVID PtrID = getSPIRVValue(PtrVal);
  if (!PtrVal->getType()->isPointerTy()) {
    return PtrID;
  }
  if (Option::UntypedPointerAddressSpace(
          PtrVal->getType()->getPointerAddressSpace())) {
    return PtrID;
  }
  SPIRVID ExpectedPtrTypeID =
      getSPIRVPointerType(PtrVal->getType(), ExpectedPointeeTy);
  auto it = IDTypeMap.find(PtrID.get());
  if (it != IDTypeMap.end() && it->second.isValid() &&
      it->second != ExpectedPtrTypeID) {
    auto storage_class =
        GetStorageClass(PtrVal->getType()->getPointerAddressSpace());
    if (storage_class == spv::StorageClassPhysicalStorageBuffer) {
      SPIRVOperandVec Ops;
      Ops << ExpectedPtrTypeID << PtrID;
      return addSPIRVInst(spv::OpBitcast, Ops);
    }
  }
  return PtrID;
}

void SPIRVProducerPassImpl::GenerateSamplers() {
  SamplerLiteralToIDMap.clear();
  DenseMap<unsigned, unsigned> SamplerLiteralToDescriptorSetMap;
  DenseMap<unsigned, unsigned> SamplerLiteralToBindingMap;

  // We might have samplers in the sampler map that are not used
  // in the translation unit.  We need to allocate variables
  // for them and bindings too.
  DenseSet<unsigned> used_bindings;

  auto *var_fn = module->getFunction(clspv::LiteralSamplerFunction());
  // Return if there are no literal samplers.
  if (!var_fn)
    return;

  for (auto user : var_fn->users()) {
    // Populate SamplerLiteralToDescriptorSetMap and
    // SamplerLiteralToBindingMap.
    //
    // Look for calls like
    //   call %opencl.sampler_t addrspace(2)*
    //       @clspv.sampler.var.literal(
    //          i32 descriptor,
    //          i32 binding,
    //          i32 sampler_mask,
    //          <sampler type> zeroinitializer)
    if (auto *call = dyn_cast<CallInst>(user)) {
      const auto sampler_value = static_cast<unsigned>(
          dyn_cast<ConstantInt>(
              call->getArgOperand(ClspvOperand::kSamplerParams))
              ->getZExtValue());
      const auto descriptor_set = static_cast<unsigned>(
          dyn_cast<ConstantInt>(
              call->getArgOperand(ClspvOperand::kSamplerDescriptorSet))
              ->getZExtValue());
      const auto binding = static_cast<unsigned>(
          dyn_cast<ConstantInt>(
              call->getArgOperand(ClspvOperand::kSamplerBinding))
              ->getZExtValue());

      SamplerLiteralToDescriptorSetMap[sampler_value] = descriptor_set;
      SamplerLiteralToBindingMap[sampler_value] = binding;
      used_bindings.insert(binding);
    }
  }

  DenseSet<size_t> seen;
  for (auto user : var_fn->users()) {
    if (!isa<CallInst>(user))
      continue;

    auto call = cast<CallInst>(user);
    const unsigned sampler_value = static_cast<unsigned>(
        dyn_cast<ConstantInt>(call->getArgOperand(ClspvOperand::kSamplerParams))
            ->getZExtValue());

    // Already allocated a variable for this value.
    if (!seen.insert(sampler_value).second)
      continue;

    // Add literal samplers to each entry point interface as an
    // over-approximation.
    auto sampler_type_id = getSPIRVPointerType(SamplerPointerTy, SamplerDataTy);
    auto sampler_var_id = addSPIRVGlobalVariable(
        sampler_type_id, spv::StorageClassUniformConstant,
        /* DataTypeID = */ SPIRVID(0),
        /* InitId = */ SPIRVID(0), /* add_interface = */ true);

    SamplerLiteralToIDMap[sampler_value] = sampler_var_id;

    unsigned descriptor_set;
    unsigned binding;
    if (SamplerLiteralToBindingMap.find(sampler_value) ==
        SamplerLiteralToBindingMap.end()) {
      // This sampler is not actually used.  Find the next one.
      for (binding = 0; used_bindings.count(binding); binding++) {
      }
      descriptor_set = 0; // Literal samplers always use descriptor set 0.
      used_bindings.insert(binding);
    } else {
      descriptor_set = SamplerLiteralToDescriptorSetMap[sampler_value];
      binding = SamplerLiteralToBindingMap[sampler_value];

      auto import_id = getReflectionImport();
      SPIRVOperandVec Ops;
      Ops << getSPIRVType(Type::getVoidTy(module->getContext())) << import_id
          << reflection::ExtInstLiteralSampler
          << getSPIRVInt32Constant(descriptor_set)
          << getSPIRVInt32Constant(binding)
          << getSPIRVInt32Constant(sampler_value);
      addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
    }

    // Ops[0] = Target ID
    // Ops[1] = Decoration (DescriptorSet)
    // Ops[2] = LiteralNumber according to Decoration
    SPIRVOperandVec Ops;
    Ops << sampler_var_id << spv::DecorationDescriptorSet << descriptor_set;

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    // Ops[0] = Target ID
    // Ops[1] = Decoration (Binding)
    // Ops[2] = LiteralNumber according to Decoration
    Ops.clear();
    Ops << sampler_var_id << spv::DecorationBinding << binding;

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  }
}

void SPIRVProducerPassImpl::GenerateResourceVars() {
  ValueMapType &VMap = getValueMap();

  // Generate variables.  Make one for each of resource var info object.
  for (auto *info : ModuleOrderedResourceVars) {
    Type *type = info->var_fn->getReturnType();
    // Remap the address space for opaque types.
    switch (info->arg_kind) {
    case clspv::ArgKind::Sampler:
    case clspv::ArgKind::SampledImage:
    case clspv::ArgKind::StorageImage:
    case clspv::ArgKind::StorageTexelBuffer:
    case clspv::ArgKind::UniformTexelBuffer:
      type =
          PointerType::get(module->getContext(), AddressSpace::UniformConstant);
      break;
    default:
      break;
    }

    const auto sc = GetStorageClassForArgKind(info->arg_kind);

    auto type_id = getSPIRVPointerType(type, info->data_type);
    SPIRVID data_ty_id;
    if (UntypedPointerStorageClass(sc)) {
      data_ty_id = getSPIRVType(info->data_type, true);
    }
    info->var_id = addSPIRVGlobalVariable(type_id, sc, data_ty_id);

    // Map calls to the variable-builtin-function.
    for (auto &U : info->var_fn->uses()) {
      if (auto *call = dyn_cast<CallInst>(U.getUser())) {
        const auto set = unsigned(
            dyn_cast<ConstantInt>(call->getOperand(0))->getZExtValue());
        const auto binding = unsigned(
            dyn_cast<ConstantInt>(call->getOperand(1))->getZExtValue());
        if (set == info->descriptor_set && binding == info->binding) {
          switch (info->arg_kind) {
          case clspv::ArgKind::Buffer:
          case clspv::ArgKind::BufferUBO:
          case clspv::ArgKind::Pod:
          case clspv::ArgKind::PodUBO:
          case clspv::ArgKind::PointerUBO:
          case clspv::ArgKind::PodPushConstant:
            // The call maps to the variable directly.
            VMap[call] = info->var_id;
            break;
          case clspv::ArgKind::Sampler:
          case clspv::ArgKind::SampledImage:
          case clspv::ArgKind::StorageImage:
          case clspv::ArgKind::StorageTexelBuffer:
          case clspv::ArgKind::UniformTexelBuffer:
            // The call maps to a load we generate later.
            ResourceVarDeferredLoadCalls[call] = info->var_id;
            break;
          default:
            llvm_unreachable("Unhandled arg kind");
          }
        }
      }
    }
  }

  // Generate associated decorations.
  SPIRVOperandVec Ops;
  for (auto *info : ModuleOrderedResourceVars) {
    // Push constants don't need descriptor set or binding decorations.
    if (info->arg_kind == clspv::ArgKind::PodPushConstant)
      continue;

    // Decorate with DescriptorSet and Binding.
    Ops.clear();
    Ops << info->var_id << spv::DecorationDescriptorSet << info->descriptor_set;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    Ops.clear();
    Ops << info->var_id << spv::DecorationBinding << info->binding;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    if (info->coherent) {
      // Decorate with Coherent if required for the variable.
      Ops.clear();
      Ops << info->var_id << spv::DecorationCoherent;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
    }

    // Generate NonWritable and NonReadable
    switch (info->arg_kind) {
    case clspv::ArgKind::Buffer:
    case clspv::ArgKind::BufferUBO:
      if (info->var_fn->getReturnType()->getPointerAddressSpace() ==
          clspv::AddressSpace::Constant) {
        Ops.clear();
        Ops << info->var_id << spv::DecorationNonWritable;
        addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
      }
      break;
    case clspv::ArgKind::StorageImage: {
      // TODO(alan-baker): This is conservative. If compiling for OpenCL 2.0 or
      // above, the compiler treats all write_only images as read_write images.
      if (IsWriteOnlyImageType(info->data_type)) {
        Ops.clear();
        Ops << info->var_id << spv::DecorationNonReadable;
        addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
      }
      break;
    }
    default:
      break;
    }
  }
}

void SPIRVProducerPassImpl::GenerateGlobalVar(GlobalVariable &GV) {
  ValueMapType &VMap = getValueMap();
  std::vector<SPIRVID> &BuiltinDimVec = getBuiltinDimVec();
  const DataLayout &DL = GV.getParent()->getDataLayout();

  const spv::BuiltIn BuiltinType = GetBuiltin(GV.getName());
  Type *Ty = GV.getType();
  PointerType *PTy = cast<PointerType>(Ty);

  SPIRVID InitializerID;

  // Workgroup size is handled differently (it goes into a constant)
  if (spv::BuiltInWorkgroupSize == BuiltinType) {
    uint32_t PrevXDimCst = 0xFFFFFFFF;
    uint32_t PrevYDimCst = 0xFFFFFFFF;
    uint32_t PrevZDimCst = 0xFFFFFFFF;
    bool HasMD = true;
    for (Function &Func : *GV.getParent()) {
      if (Func.isDeclaration()) {
        continue;
      }

      // We only need to check kernels.
      if (Func.getCallingConv() != CallingConv::SPIR_KERNEL) {
        continue;
      }

      if (const MDNode *MD =
              dyn_cast<Function>(&Func)->getMetadata("reqd_work_group_size")) {
        uint32_t CurXDimCst = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(0))->getZExtValue());
        uint32_t CurYDimCst = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(1))->getZExtValue());
        uint32_t CurZDimCst = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue());

        if (PrevXDimCst == 0xFFFFFFFF && PrevYDimCst == 0xFFFFFFFF &&
            PrevZDimCst == 0xFFFFFFFF) {
          PrevXDimCst = CurXDimCst;
          PrevYDimCst = CurYDimCst;
          PrevZDimCst = CurZDimCst;
        } else if (CurXDimCst != PrevXDimCst || CurYDimCst != PrevYDimCst ||
                   CurZDimCst != PrevZDimCst) {
          HasMD = false;
          continue;
        } else {
          continue;
        }

        //
        // Generate OpConstantComposite.
        //
        // Ops[0] : Result Type ID
        // Ops[1] : Constant size for x dimension.
        // Ops[2] : Constant size for y dimension.
        // Ops[3] : Constant size for z dimension.
        SPIRVOperandVec Ops;

        SPIRVID XDimCstID =
            getSPIRVValue(mdconst::extract<ConstantInt>(MD->getOperand(0)));
        SPIRVID YDimCstID =
            getSPIRVValue(mdconst::extract<ConstantInt>(MD->getOperand(1)));
        SPIRVID ZDimCstID =
            getSPIRVValue(mdconst::extract<ConstantInt>(MD->getOperand(2)));

        Ops << GV.getValueType() << XDimCstID << YDimCstID
            << ZDimCstID;

        InitializerID =
            addSPIRVInst<kGlobalVariables>(spv::OpConstantComposite, Ops);
      } else {
        HasMD = false;
      }
    }

    // If all kernels do not have metadata for reqd_work_group_size, generate
    // OpSpecConstants for x/y/z dimension.
    if (!HasMD || clspv::Option::NonUniformNDRangeSupported()) {
      //
      // Generate OpSpecConstants for x/y/z dimension.
      //
      // Ops[0] : Result Type ID
      // Ops[1] : Constant size for x/y/z dimension (Literal Number).

      // Allocate spec constants for workgroup size.
      clspv::AddWorkgroupSpecConstants(module.get());

      SPIRVOperandVec Ops;
      SPIRVID result_type_id = getSPIRVType(
          dyn_cast<VectorType>(GV.getValueType())->getElementType());

      // X Dimension
      Ops << result_type_id << 1;
      SPIRVID XDimCstID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

      // Y Dimension
      Ops.clear();
      Ops << result_type_id << 1;
      SPIRVID YDimCstID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

      // Z Dimension
      Ops.clear();
      Ops << result_type_id << 1;
      SPIRVID ZDimCstID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

      BuiltinDimVec.push_back(XDimCstID);
      BuiltinDimVec.push_back(YDimCstID);
      BuiltinDimVec.push_back(ZDimCstID);

      //
      // Generate OpSpecConstantComposite.
      //
      // Ops[0] : Result Type ID
      // Ops[1] : Constant size for x dimension.
      // Ops[2] : Constant size for y dimension.
      // Ops[3] : Constant size for z dimension.
      Ops.clear();
      Ops << GV.getValueType() << XDimCstID << YDimCstID << ZDimCstID;

      InitializerID =
          addSPIRVInst<kConstants>(spv::OpSpecConstantComposite, Ops);
    }
  } else if (BuiltinType == spv::BuiltInWorkDim) {
    // 1. Generate a specialization constant with a default of 3.
    // 2. Allocate and annotate a SpecId for the constant.
    // 3. Use the spec constant as the initializer for the variable.
    SPIRVOperandVec Ops;

    //
    // Generate OpSpecConstant.
    //
    // Ops[0] : Result Type ID
    // Ops[1] : Default literal value

    Ops << IntegerType::get(GV.getContext(), 32) << 3;

    InitializerID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    //
    // Generate SpecId decoration.
    //
    // Ops[0] : target
    // Ops[1] : decoration
    // Ops[2] : SpecId
    auto spec_id = AllocateSpecConstant(module.get(), SpecConstant::kWorkDim);
    Ops.clear();
    Ops << InitializerID << spv::DecorationSpecId << spec_id;

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  } else if (BuiltinType == spv::BuiltInGlobalOffset) {
    // 1. Generate a spec constant with a default of {0, 0, 0}.
    // 2. Allocate and annotate SpecIds for the constants.
    // 3. Use the spec constant as the initializer for the variable.
    SPIRVOperandVec Ops;

    //
    // Generate OpSpecConstant for each dimension.
    //
    // Ops[0] : Result Type ID
    // Ops[1] : Default literal value
    //
    Ops << IntegerType::get(GV.getContext(), 32) << 0;
    SPIRVID x_id = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    Ops.clear();
    Ops << IntegerType::get(GV.getContext(), 32) << 0;
    SPIRVID y_id = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    Ops.clear();
    Ops << IntegerType::get(GV.getContext(), 32) << 0;
    SPIRVID z_id = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    //
    // Generate SpecId decoration for each dimension.
    //
    // Ops[0] : target
    // Ops[1] : decoration
    // Ops[2] : SpecId
    //
    auto spec_id =
        AllocateSpecConstant(module.get(), SpecConstant::kGlobalOffsetX);
    Ops.clear();
    Ops << x_id << spv::DecorationSpecId << spec_id;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    spec_id = AllocateSpecConstant(module.get(), SpecConstant::kGlobalOffsetY);
    Ops.clear();
    Ops << y_id << spv::DecorationSpecId << spec_id;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    spec_id = AllocateSpecConstant(module.get(), SpecConstant::kGlobalOffsetZ);
    Ops.clear();
    Ops << z_id << spv::DecorationSpecId << spec_id;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    //
    // Generate OpSpecConstantComposite.
    //
    // Ops[0] : type id
    // Ops[1..n-1] : elements
    //
    Ops.clear();
    Ops << GV.getValueType() << x_id << y_id << z_id;
    InitializerID = addSPIRVInst<kConstants>(spv::OpSpecConstantComposite, Ops);
  } else if (BuiltinType == spv::BuiltInSubgroupMaxSize) {
    // 1. Generate a specialization constant with a default of 1.
    // 2. Allocate and annotate a SpecId for the constant.
    // 3. Use the spec constant as the initializer for the variable.
    SPIRVOperandVec Ops;

    //
    // Generate OpSpecConstant.
    //
    // Ops[0] : Result Type ID
    // Ops[1] : Default literal value

    Ops << IntegerType::get(GV.getContext(), 32) << 1;

    InitializerID = addSPIRVInst<kConstants>(spv::OpSpecConstant, Ops);

    //
    // Generate SpecId decoration.
    //
    // Ops[0] : target
    // Ops[1] : decoration
    // Ops[2] : SpecId
    auto spec_id =
        AllocateSpecConstant(module.get(), SpecConstant::kSubgroupMaxSize);
    Ops.clear();
    Ops << InitializerID << spv::DecorationSpecId << spec_id;

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  }

  const auto AS = PTy->getAddressSpace();
  const auto spvSC = GetStorageClass(AS);

  const bool module_scope_constant_external_init =
      (AS == AddressSpace::Constant) && GV.hasInitializer() &&
      clspv::Option::ModuleConstantsInStorageBuffer();

  const bool is_printf_buffer = GV.getName() == PrintfBufferVariableName();

  if (GV.hasInitializer()) {
    auto GVInit = GV.getInitializer();
    if (!isa<UndefValue>(GVInit) && !module_scope_constant_external_init) {
      InitializerID = getSPIRVValue(GVInit);
    }
  }

  // All private, module private, and local global variables can be added to
  // interfaces conservatively.
  const bool interface =
      (AS == AddressSpace::Private || AS == AddressSpace::ModuleScopePrivate ||
       AS == AddressSpace::Local);
  SPIRVID data_ty_id;
  if (Option::UntypedPointerAddressSpace(AS)) {
    data_ty_id = getSPIRVType(GV.getValueType(), true);
    if (AS == AddressSpace::Local) {
      auto *StructTy = StructType::get(GV.getValueType());
      TypesNeedingLayout.insert(StructTy);
      data_ty_id = getSPIRVType(StructTy, true);

      // Add block decoration
      SPIRVOperandVec Ops;
      Ops << data_ty_id << spv::DecorationBlock;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
    }
  }
  auto ptr_id = getSPIRVPointerType(Ty, GV.getValueType());

  SPIRVID var_id;
  if (!(module_scope_constant_external_init &&
        clspv::Option::PhysicalStorageBuffers())) {
    var_id = addSPIRVGlobalVariable(ptr_id, spvSC, data_ty_id, InitializerID,
                                    interface);
    if (spvSC == spv::StorageClass::StorageClassWorkgroup) {
      WorkgroupVariableSizeMap.insert(std::make_pair(
          var_id.get(),
          BitcastUtils::SizeInBits(DL, GV.getValueType()) / CHAR_BIT));
    }
    VMap[&GV] = var_id;
  }

  auto IsOpenCLBuiltin = [](spv::BuiltIn builtin) {
    return builtin == spv::BuiltInWorkDim ||
           builtin == spv::BuiltInGlobalOffset ||
           builtin == spv::BuiltInSubgroupMaxSize;
  };

  // If we have a builtin (not an OpenCL builtin).
  if (spv::BuiltInMax != BuiltinType && !IsOpenCLBuiltin(BuiltinType)) {
    //
    // Generate OpDecorate.
    //
    // DOps[0] = Target ID
    // DOps[1] = Decoration (Builtin)
    // DOps[2] = BuiltIn ID
    SPIRVID ResultID;

    // WorkgroupSize is different, we decorate the constant composite that has
    // its value, rather than the variable that we use to access the value.
    if (spv::BuiltInWorkgroupSize == BuiltinType) {
      ResultID = InitializerID;
      // Save both the value and variable IDs for later.
      WorkgroupSizeValueID = InitializerID;
      WorkgroupSizeVarID = getSPIRVValue(&GV);
    } else {
      ResultID = getSPIRVValue(&GV);
    }

    SPIRVOperandVec Ops;
    Ops << ResultID << spv::DecorationBuiltIn << BuiltinType;

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  } else if (module_scope_constant_external_init) {
    // Emit the intializer as a reflection instruction.
    std::string hexbytes;
    llvm::raw_string_ostream str(hexbytes);
    clspv::ConstantEmitter(DL, str).Emit(GV.getInitializer());
    auto data_id = addSPIRVInst<kDebug>(spv::OpString, str.str().c_str());
    SPIRVOperandVec Ops;
    // If using physical storage buffers, lower the constants GV as a push
    // constant containing a pointer, otherwise use a storage buffer
    if (clspv::Option::PhysicalStorageBuffers()) {
      std::string hexbytes;
      llvm::raw_string_ostream str(hexbytes);
      clspv::ConstantEmitter(DL, str).Emit(GV.getInitializer());

      auto PushConstGV =
          module->getGlobalVariable(clspv::PushConstantsVariableName());
      auto STy = cast<StructType>(PushConstGV->getValueType());
      auto MD = PushConstGV->getMetadata(clspv::PushConstantsMetadataName());
      bool Found [[maybe_unused]] = false;
      uint32_t Offset = 0;

      // Find the push constant offset for the module constants pointer
      for (unsigned i = 0; i < STy->getNumElements(); i++) {
        auto pc = static_cast<clspv::PushConstant>(
            mdconst::extract<ConstantInt>(MD->getOperand(i))->getZExtValue());

        if (pc == clspv::PushConstant::ModuleConstantsPointer) {
          Found = true;
          Offset = GetExplicitLayoutStructMemberOffset(STy, i, DL);
        }
      }
      assert(Found);

      Ops << getSPIRVType(Type::getVoidTy(module->getContext()))
          << getReflectionImport()
          << reflection::ExtInstConstantDataPointerPushConstant
          << getSPIRVInt32Constant(Offset) << getSPIRVInt32Constant(8)
          << data_id;
      addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
    } else {
      // This module scope constant is initialized from a storage buffer with
      // data provided by the host at binding 0 of the next descriptor set.
      const uint32_t descriptor_set = TakeDescriptorIndex(module.get());

      // Reflection instruction for constant data.
      Ops << getSPIRVType(Type::getVoidTy(module->getContext()))
          << getReflectionImport()
          << reflection::ExtInstConstantDataStorageBuffer
          << getSPIRVInt32Constant(descriptor_set) << getSPIRVInt32Constant(0)
          << data_id;
      addSPIRVInst<kReflection>(spv::OpExtInst, Ops);

      // OpDecorate %var DescriptorSet <descriptor_set>
      Ops.clear();
      Ops << VMap[&GV] << spv::DecorationDescriptorSet << descriptor_set;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

      // OpDecorate %var Binding <binding>
      Ops.clear();
      Ops << VMap[&GV] << spv::DecorationBinding << 0;
      addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
    }
  } else if (is_printf_buffer) {
    const uint32_t descriptor_set = TakeDescriptorIndex(module.get());
    SPIRVOperandVec Ops;
    Ops << var_id << spv::DecorationDescriptorSet << descriptor_set;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    Ops.clear();
    Ops << var_id << spv::DecorationBinding << 0;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    Ops.clear();
    Ops << getSPIRVType(Type::getVoidTy(module->getContext()))
        << getReflectionImport() << reflection::ExtInstPrintfBufferStorageBuffer
        << getSPIRVInt32Constant(descriptor_set) << getSPIRVInt32Constant(0)
        << getSPIRVInt32Constant(clspv::Option::PrintfBufferSize());
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
}

void SPIRVProducerPassImpl::GenerateFuncPrologue(Function &F) {
  ValueMapType &VMap = getValueMap();
  EntryPointVecType &EntryPoints = getEntryPointVec();
  auto &GlobalConstFuncTyMap = getGlobalConstFuncTypeMap();

  FunctionType *FTy = F.getFunctionType();
  if (F.getCallingConv() != CallingConv::SPIR_KERNEL) {
    auto where = GlobalConstFuncTyMap.find(FTy);
    if (where != GlobalConstFuncTyMap.end()) {
      FTy = where->second.first;
    }
  } else {
    SmallVector<Type *, 4> NewFuncParamTys;
    FunctionType *NewFTy =
        FunctionType::get(FTy->getReturnType(), NewFuncParamTys, false);
    FTy = NewFTy;
  }

  //
  // Generate OPFunction.
  //

  // FOps[0] : Result Type ID
  // FOps[1] : Function Control
  // FOps[2] : Function Type ID
  SPIRVOperandVec FOps;

  Type *inferred_ret_ty = nullptr;
  SmallVector<Type *, 8> inferred_param_tys;
  if (F.getCallingConv() != CallingConv::SPIR_KERNEL) {
    for (auto *user : F.users()) {
      if (isa<CallInst>(user)) {
        inferred_ret_ty =
            InferType(user, module->getContext(), &InferredTypeCache);
        if (inferred_ret_ty)
          break;
      }
    }
    for (auto &Arg : F.args()) {
      inferred_param_tys.push_back(
          InferType(&Arg, module->getContext(), &InferredTypeCache));
    }
  } else {
    inferred_ret_ty = FTy->getReturnType();
  }

  SPIRVID ret_type_id =
      getSPIRVPointerType(FTy->getReturnType(), inferred_ret_ty);

  // Find SPIRV instruction for return type.
  FOps << ret_type_id;

  // Check function attributes for SPIRV Function Control.
  uint32_t FuncControl = spv::FunctionControlMaskNone;
  if (F.hasFnAttribute(Attribute::AlwaysInline)) {
    FuncControl |= spv::FunctionControlInlineMask;
  }
  if (F.hasFnAttribute(Attribute::NoInline)) {
    FuncControl |= spv::FunctionControlDontInlineMask;
  }
  // TODO: Check llvm attribute for Function Control Pure.
  if (F.onlyReadsMemory()) {
    FuncControl |= spv::FunctionControlPureMask;
  }
  // TODO: Check llvm attribute for Function Control Const.
  if (F.doesNotAccessMemory()) {
    FuncControl |= spv::FunctionControlConstMask;
  }

  FOps << FuncControl;

  SPIRVID FTyID =
      getSPIRVFunctionType(FTy, inferred_ret_ty, inferred_param_tys);

  FOps << FTyID;

  // Generate SPIRV instruction for function.
  SPIRVID FID = addSPIRVInst(spv::OpFunction, FOps);
  VMap[&F] = FID;

  if (F.getCallingConv() == CallingConv::SPIR_KERNEL) {
    EntryPoints.push_back(std::make_pair(&F, FID));
  }

  if (clspv::Option::ShowIDs()) {
    errs() << "Function " << F.getName() << " is " << FID.get() << "\n";
  }

  //
  // Generate OpFunctionParameter for Normal function.
  //
  if (F.getCallingConv() != CallingConv::SPIR_KERNEL) {

    // Iterate Argument for name instead of param type from function type.
    unsigned ArgIdx = 0;
    for (Argument &Arg : F.args()) {
      // ParamOps[0] : Result Type ID
      SPIRVOperandVec Ops;

      // Find SPIRV instruction for parameter type.
      SPIRVID ParamTyID =
          getSPIRVPointerType(Arg.getType(), inferred_param_tys[ArgIdx]);
      Ops << ParamTyID;

      // Generate SPIRV instruction for parameter.
      SPIRVID param_id = addSPIRVInst(spv::OpFunctionParameter, Ops);
      VMap[&Arg] = param_id;

      if (CalledWithCoherentResource(Arg)) {
        // If the arg is passed a coherent resource ever, then decorate this
        // parameter with Coherent too.
        Ops.clear();
        Ops << param_id << spv::DecorationCoherent;
        addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
      }

      // PhysicalStorageBuffer args require a Restrict or Aliased decoration
      if (auto *PtrTy = dyn_cast<PointerType>(Arg.getType())) {
        if (clspv::Option::PhysicalStorageBuffers() &&
            (PtrTy->getAddressSpace() == clspv::AddressSpace::Global ||
             PtrTy->getAddressSpace() == clspv::AddressSpace::Constant)) {
          Ops.clear();
          Ops << param_id
              << (Arg.hasNoAliasAttr() ? spv::DecorationRestrict
                                       : spv::DecorationAliased);
          addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
        }
      }

      ArgIdx++;
    }
  }
}

void SPIRVProducerPassImpl::GenerateModuleInfo() {
  EntryPointVecType &EntryPoints = getEntryPointVec();
  auto &EntryPointInterfaces = getEntryPointInterfacesList();
  std::vector<SPIRVID> &BuiltinDimVec = getBuiltinDimVec();

  // Add denorm capabilities if needed
  if ((ExecutionModeDenorm(FloatingPointType::fp16) == DenormMode::preserve) ||
      (ExecutionModeDenorm(FloatingPointType::fp32) == DenormMode::preserve) ||
      (ExecutionModeDenorm(FloatingPointType::fp64) == DenormMode::preserve)) {
    addCapability(spv::CapabilityDenormPreserve);
  }
  if ((ExecutionModeDenorm(FloatingPointType::fp16) ==
       DenormMode::flush_to_zero) ||
      (ExecutionModeDenorm(FloatingPointType::fp32) ==
       DenormMode::flush_to_zero) ||
      (ExecutionModeDenorm(FloatingPointType::fp64) ==
       DenormMode::flush_to_zero)) {
    addCapability(spv::CapabilityDenormFlushToZero);
  }
  if (ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp16) ||
      ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp32) ||
      ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp64)) {
    addCapability(spv::CapabilitySignedZeroInfNanPreserve);
  }

  SPIRVOperandVec Ops;

  for (auto Capability : CapabilitySet) {
    //
    // Generate OpCapability
    //
    // Ops[0] = Capability
    addSPIRVInst<kCapabilities>(spv::OpCapability, Capability);
  }

  // Storage buffer and variable pointer extensions were made core in SPIR-V
  // 1.3.
  if (SpvVersion() < SPIRVVersion::SPIRV_1_3) {
    //
    // Generate OpExtension.
    //
    // Ops[0] = Name (Literal String)
    //
    addSPIRVInst<kExtensions>(spv::OpExtension,
                              "SPV_KHR_storage_buffer_storage_class");

    if (hasVariablePointers() || hasVariablePointersStorageBuffer()) {
      //
      // Generate OpExtension.
      //
      // Ops[0] = Name (Literal String)
      //
      addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_variable_pointers");
    }
  }

  bool hasDenorms =
      (clspv::Option::ExecutionModeDenorm(FloatingPointType::fp16) !=
       DenormMode::unspecified) ||
      (clspv::Option::ExecutionModeDenorm(FloatingPointType::fp32) !=
       DenormMode::unspecified) ||
      (clspv::Option::ExecutionModeDenorm(FloatingPointType::fp64) !=
       DenormMode::unspecified);

  bool hasSignedZero =
      ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp16) ||
      ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp32) ||
      ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp64);

  if (SpvVersion() < SPIRVVersion::SPIRV_1_4 &&
      (hasConvertToF() || hasDenorms || hasSignedZero)) {
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_float_controls");
  }

  if (clspv::Option::PhysicalStorageBuffers()) {
    addSPIRVInst<kExtensions>(spv::OpExtension,
                              "SPV_KHR_physical_storage_buffer");
  }

  // Descriptor indexing extension was made core in SPIR-V 1.5.
  if (hasNonUniformPointers() &&
      clspv::Option::SpvVersion() < clspv::Option::SPIRVVersion::SPIRV_1_5) {
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_EXT_descriptor_indexing");
  }

  if (SpvVersion() < SPIRVVersion::SPIRV_1_6 && hasIntegerDot()) {
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_integer_dot_product");
  }

  if (Option::UntypedPointers()) {
    // Workgroup explicit layout is required for untyped pointers in workgroup
    // memory.
    if (CapabilitySet.count(spv::CapabilityWorkgroupMemoryExplicitLayoutKHR)) {
      addSPIRVInst<kExtensions>(spv::OpExtension,
                                "SPV_KHR_workgroup_memory_explicit_layout");
    }
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_untyped_pointers");
  }

  if (CapabilitySet.count(spv::CapabilityFMAKHR)) {
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_fma");
  }

  //
  // Generate OpMemoryModel
  //
  // Memory model for Vulkan will always be GLSL450.

  // Ops[0] = Addressing Model
  // Ops[1] = Memory Model
  Ops.clear();
  Ops << (clspv::Option::PhysicalStorageBuffers()
              ? spv::AddressingModelPhysicalStorageBuffer64
              : spv::AddressingModelLogical)
      << spv::MemoryModelGLSL450;

  addSPIRVInst<kMemoryModel>(spv::OpMemoryModel, Ops);

  //
  // Generate OpEntryPoint
  //
  for (auto EntryPoint : EntryPoints) {
    // Ops[0] = Execution Model
    // Ops[1] = EntryPoint ID
    // Ops[2] = Name (Literal String)
    // ...
    //
    // TODO: Do we need to consider Interface ID for forward references???
    Ops.clear();
    const StringRef &name = EntryPoint.first->getName();
    Ops << spv::ExecutionModelGLCompute << EntryPoint.second << name;

    for (auto &Interface : EntryPointInterfaces) {
      Ops << Interface;
    }

    // Starting in SPIR-V 1.4, all statically used global variables must be
    // included in the interface. Private and statically-sized workgroup
    // variables are added to all entry points. Kernel arguments are handled
    // here.
    if (SpvVersion() >= SPIRVVersion::SPIRV_1_4) {
      auto *F = dyn_cast<Function>(EntryPoint.first);
      assert(F);
      assert(F->getCallingConv() == CallingConv::SPIR_KERNEL);

      auto &resource_var_at_index = FunctionToResourceVarsMap[F];
      for (auto *info : resource_var_at_index) {
        if (info) {
          Ops << info->var_id;
        }
      }

      if (clspv::Option::ModuleConstantsInStorageBuffer() &&
          !clspv::Option::PhysicalStorageBuffers()) {
        auto *V = module->getGlobalVariable(
            clspv::ClusteredConstantsVariableName(), true);
        if (V) {
          Ops << getValueMap()[V];
        }
      }

      if (auto *V = module->getGlobalVariable(clspv::PrintfBufferVariableName(),
                                              true)) {
        Ops << getValueMap()[V];
      }

      auto local_spec_id_md =
          module->getNamedMetadata(clspv::LocalSpecIdMetadataName());
      if (local_spec_id_md) {
        for (auto spec_id_op : local_spec_id_md->operands()) {
          if (dyn_cast<Function>(
                  dyn_cast<ValueAsMetadata>(spec_id_op->getOperand(0))
                      ->getValue()) == F) {
            int64_t spec_id =
                mdconst::extract<ConstantInt>(spec_id_op->getOperand(2))
                    ->getSExtValue();
            if (spec_id > 0) {
              auto &info = LocalSpecIdInfoMap[spec_id];
              Ops << info.variable_id;
            }
          }
        }
      }

      // If the kernel uses the global push constant interface it will not be
      // covered by the resource variable iteration above.
      if (auto *PC =
              module->getGlobalVariable(clspv::PushConstantsVariableName())) {
        assert(PC);
        Ops << getValueMap()[PC];
      }
    }

    addSPIRVInst<kEntryPoints>(spv::OpEntryPoint, Ops);
  }

  if (BuiltinDimVec.empty()) {
    for (auto EntryPoint : EntryPoints) {
      const MDNode *MD = dyn_cast<Function>(EntryPoint.first)
                             ->getMetadata("reqd_work_group_size");
      if ((MD != nullptr) && !clspv::Option::NonUniformNDRangeSupported()) {
        //
        // Generate OpExecutionMode
        //

        // Ops[0] = Entry Point ID
        // Ops[1] = Execution Mode
        // Ops[2] ... Ops[n] = Optional literals according to Execution Mode
        Ops.clear();
        Ops << EntryPoint.second << spv::ExecutionModeLocalSize;

        uint32_t XDim = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(0))->getZExtValue());
        uint32_t YDim = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(1))->getZExtValue());
        uint32_t ZDim = static_cast<uint32_t>(
            mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue());

        Ops << XDim << YDim << ZDim;

        addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
      }
    }
  }

  // libclc expects the default rounding mode to be RTE
  if (hasConvertToF()) {
    for (auto EntryPoint : EntryPoints) {
      if (clspv::Option::FP16() &&
          ExecutionModeRoundingModeRTE(FloatingPointType::fp16)) {
        Ops.clear();
        Ops << EntryPoint.second << spv::ExecutionModeRoundingModeRTE << 16;
        addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
      }
      if (ExecutionModeRoundingModeRTE(FloatingPointType::fp32)) {
        Ops.clear();
        Ops << EntryPoint.second << spv::ExecutionModeRoundingModeRTE << 32;
        addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
      }
      if (clspv::Option::FP64() &&
          ExecutionModeRoundingModeRTE(FloatingPointType::fp64)) {
        Ops.clear();
        Ops << EntryPoint.second << spv::ExecutionModeRoundingModeRTE << 64;
        addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
      }
    }
  }

  // Emit denorm execution modes
  for (auto EntryPoint : EntryPoints) {
    auto mode = ExecutionModeDenorm(FloatingPointType::fp16);
    if (FP16() && mode == DenormMode::preserve) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormPreserve << 16;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    } else if (FP16() && mode == DenormMode::flush_to_zero) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormFlushToZero << 16;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }
    mode = ExecutionModeDenorm(FloatingPointType::fp32);
    if (mode == DenormMode::preserve) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormPreserve << 32;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    } else if (mode == DenormMode::flush_to_zero) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormFlushToZero << 32;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }
    mode = ExecutionModeDenorm(FloatingPointType::fp64);
    if (FP64() && mode == DenormMode::preserve) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormPreserve << 64;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    } else if (FP64() && mode == DenormMode::flush_to_zero) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeDenormFlushToZero << 64;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }

    // Emit SignedZeroInfNanPreserve execution modes
    if (FP16() &&
        ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp16)) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeSignedZeroInfNanPreserve
          << 16;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }
    if (ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp32)) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeSignedZeroInfNanPreserve
          << 32;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }
    if (FP64() &&
        ExecutionModeSignedZeroInfNanPreserve(FloatingPointType::fp64)) {
      Ops.clear();
      Ops << EntryPoint.second << spv::ExecutionModeSignedZeroInfNanPreserve
          << 64;
      addSPIRVInst<kExecutionModes>(spv::OpExecutionMode, Ops);
    }
  }

  //
  // Generate OpSource.
  //
  // Ops[0] = SourceLanguage ID
  // Ops[1] = Version (LiteralNum)
  //
  uint32_t LangID = spv::SourceLanguageUnknown;
  uint32_t LangVer = 0;
  switch (clspv::Option::Language()) {
  case clspv::Option::SourceLanguage::OpenCL_C_10:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 100;
    break;
  case clspv::Option::SourceLanguage::OpenCL_C_11:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 110;
    break;
  case clspv::Option::SourceLanguage::OpenCL_C_12:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 120;
    break;
  case clspv::Option::SourceLanguage::OpenCL_C_20:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 200;
    break;
  case clspv::Option::SourceLanguage::OpenCL_C_30:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 300;
    break;
  case clspv::Option::SourceLanguage::OpenCL_C_31:
    LangID = spv::SourceLanguageOpenCL_C;
    LangVer = 310;
    break;
  case clspv::Option::SourceLanguage::OpenCL_CPP:
    LangID = spv::SourceLanguageOpenCL_CPP;
    LangVer = 100;
    break;
  case clspv::Option::SourceLanguage::OpenCL_CPP_2021:
    LangID = spv::SourceLanguageOpenCL_CPP;
    LangVer = 202100;
    break;
  default:
    break;
  }

  Ops.clear();
  Ops << LangID << LangVer;
  addSPIRVInst<kDebug>(spv::OpSource, Ops);

  if (!BuiltinDimVec.empty()) {
    //
    // Generate OpDecorates for x/y/z dimension.
    //
    // Ops[0] = Target ID
    // Ops[1] = Decoration (SpecId)
    // Ops[2] = Specialization Constant ID (Literal Number)

    // X Dimension
    Ops.clear();
    Ops << BuiltinDimVec[0] << spv::DecorationSpecId << 0;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    // Y Dimension
    Ops.clear();
    Ops << BuiltinDimVec[1] << spv::DecorationSpecId << 1;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);

    // Z Dimension
    Ops.clear();
    Ops << BuiltinDimVec[2] << spv::DecorationSpecId << 2;
    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  }
}

void SPIRVProducerPassImpl::GenerateEntryPointInitialStores() {
  // Work around a driver bug.  Initializers on Private variables might not
  // work. So the start of the kernel should store the initializer value to the
  // variables.  Yes, *every* entry point pays this cost if *any* entry point
  // uses this builtin.  At this point I judge this to be an acceptable tradeoff
  // of complexity vs. runtime, for a broken driver.
  // TODO(dneto): Remove this at some point once fixed drivers are widely
  // available.
  if (WorkgroupSizeVarID.isValid()) {
    assert(WorkgroupSizeValueID.isValid());

    SPIRVOperandVec Ops;
    Ops << WorkgroupSizeVarID << WorkgroupSizeValueID;

    addSPIRVInst(spv::OpStore, Ops);
  }
}

void SPIRVProducerPassImpl::GenerateFuncBody(Function &F) {
  ValueMapType &VMap = getValueMap();

  const bool IsKernel = F.getCallingConv() == CallingConv::SPIR_KERNEL;

  for (BasicBlock &BB : F) {
    // Register BasicBlock to ValueMap.

    //
    // Generate OpLabel for Basic Block.
    //
    VMap[&BB] = addSPIRVInst(spv::OpLabel);
    if (NameBasicBlocks) {
      SPIRVOperandVec Ops;
      Ops << VMap[&BB] << BB.getName();
      addSPIRVInst<kNames>(spv::OpName, Ops);
    }

    // OpVariable instructions must come first.
    for (Instruction &I : BB) {
      if (auto *alloca = dyn_cast<AllocaInst>(&I)) {
        // Allocating a pointer requires variable pointers.
        if (alloca->getAllocatedType()->isPointerTy()) {
          setVariablePointersCapabilities(
              alloca->getAllocatedType()->getPointerAddressSpace());
        }
        GenerateInstruction(I);
      }
    }

    if (&BB == &F.getEntryBlock() && IsKernel) {
      if (clspv::Option::HackInitializers()) {
        GenerateEntryPointInitialStores();
      }
    }

    for (Instruction &I : BB) {
      if (!isa<AllocaInst>(I)) {
        GenerateInstruction(I);
      }
    }
  }
}

spv::Op SPIRVProducerPassImpl::GetSPIRVCmpOpcode(CmpInst *I) {
  const std::map<CmpInst::Predicate, spv::Op> Map = {
      {CmpInst::ICMP_EQ, spv::OpIEqual},
      {CmpInst::ICMP_NE, spv::OpINotEqual},
      {CmpInst::ICMP_UGT, spv::OpUGreaterThan},
      {CmpInst::ICMP_UGE, spv::OpUGreaterThanEqual},
      {CmpInst::ICMP_ULT, spv::OpULessThan},
      {CmpInst::ICMP_ULE, spv::OpULessThanEqual},
      {CmpInst::ICMP_SGT, spv::OpSGreaterThan},
      {CmpInst::ICMP_SGE, spv::OpSGreaterThanEqual},
      {CmpInst::ICMP_SLT, spv::OpSLessThan},
      {CmpInst::ICMP_SLE, spv::OpSLessThanEqual},
      {CmpInst::FCMP_OEQ, spv::OpFOrdEqual},
      {CmpInst::FCMP_OGT, spv::OpFOrdGreaterThan},
      {CmpInst::FCMP_OGE, spv::OpFOrdGreaterThanEqual},
      {CmpInst::FCMP_OLT, spv::OpFOrdLessThan},
      {CmpInst::FCMP_OLE, spv::OpFOrdLessThanEqual},
      {CmpInst::FCMP_ONE, spv::OpFOrdNotEqual},
      {CmpInst::FCMP_UEQ, spv::OpFUnordEqual},
      {CmpInst::FCMP_UGT, spv::OpFUnordGreaterThan},
      {CmpInst::FCMP_UGE, spv::OpFUnordGreaterThanEqual},
      {CmpInst::FCMP_ULT, spv::OpFUnordLessThan},
      {CmpInst::FCMP_ULE, spv::OpFUnordLessThanEqual},
      {CmpInst::FCMP_UNE, spv::OpFUnordNotEqual}};

  assert(0 != Map.count(I->getPredicate()));

  return Map.at(I->getPredicate());
}

spv::Op SPIRVProducerPassImpl::GetSPIRVPointerCmpOpcode(CmpInst *I) {
  const std::map<CmpInst::Predicate, spv::Op> Map = {
      {CmpInst::ICMP_UGT, spv::OpSGreaterThan},
      {CmpInst::ICMP_UGE, spv::OpSGreaterThanEqual},
      {CmpInst::ICMP_ULT, spv::OpSLessThan},
      {CmpInst::ICMP_ULE, spv::OpSLessThanEqual}};

  assert(0 != Map.count(I->getPredicate()));

  return Map.at(I->getPredicate());
}

spv::Op SPIRVProducerPassImpl::GetSPIRVCastOpcode(Instruction &I) {
  const std::map<unsigned, spv::Op> Map{
      {Instruction::Trunc, spv::OpUConvert},
      {Instruction::ZExt, spv::OpUConvert},
      {Instruction::SExt, spv::OpSConvert},
      {Instruction::FPToUI, spv::OpConvertFToU},
      {Instruction::FPToSI, spv::OpConvertFToS},
      {Instruction::UIToFP, spv::OpConvertUToF},
      {Instruction::SIToFP, spv::OpConvertSToF},
      {Instruction::FPTrunc, spv::OpFConvert},
      {Instruction::FPExt, spv::OpFConvert},
      {Instruction::BitCast, spv::OpBitcast},
      {Instruction::PtrToInt, spv::OpConvertPtrToU},
      {Instruction::PtrToAddr, spv::OpConvertPtrToU},
      {Instruction::IntToPtr, spv::OpConvertUToPtr},
  };

  assert(0 != Map.count(I.getOpcode()));

  return Map.at(I.getOpcode());
}

spv::Op SPIRVProducerPassImpl::GetSPIRVBinaryOpcode(Instruction &I) {
  if (I.getType()->isIntOrIntVectorTy(1)) {
    switch (I.getOpcode()) {
    default:
      break;
    case Instruction::Or:
      return spv::OpLogicalOr;
    case Instruction::And:
      return spv::OpLogicalAnd;
    case Instruction::Xor:
      return spv::OpLogicalNotEqual;
    }
  }

  const std::map<unsigned, spv::Op> Map{
      {Instruction::Add, spv::OpIAdd},
      {Instruction::FAdd, spv::OpFAdd},
      {Instruction::Sub, spv::OpISub},
      {Instruction::FSub, spv::OpFSub},
      {Instruction::Mul, spv::OpIMul},
      {Instruction::FMul, spv::OpFMul},
      {Instruction::UDiv, spv::OpUDiv},
      {Instruction::SDiv, spv::OpSDiv},
      {Instruction::FDiv, spv::OpFDiv},
      {Instruction::URem, spv::OpUMod},
      {Instruction::FRem, spv::OpFRem},
      {Instruction::Or, spv::OpBitwiseOr},
      {Instruction::Xor, spv::OpBitwiseXor},
      {Instruction::And, spv::OpBitwiseAnd},
      {Instruction::Shl, spv::OpShiftLeftLogical},
      {Instruction::LShr, spv::OpShiftRightLogical},
      {Instruction::AShr, spv::OpShiftRightArithmetic}};

  assert(0 != Map.count(I.getOpcode()));

  return Map.at(I.getOpcode());
}

SPIRVID SPIRVProducerPassImpl::getSPIRVBuiltin(spv::BuiltIn BID, Type *Ty,
                                               spv::Capability Cap) {
  SPIRVID RID;

  auto ii = BuiltinConstantMap.find(BID);

  if (ii != BuiltinConstantMap.end()) {
    return ii->second;
  } else {
    addCapability(Cap);

    auto *ptr_ty = PointerType::get(module->getContext(), AddressSpace::Input);
    auto ptr_id = getSPIRVPointerType(ptr_ty, Ty);
    RID = addSPIRVGlobalVariable(ptr_id, spv::StorageClassInput);

    BuiltinConstantMap[BID] = RID;

    //
    // Generate OpDecorate.
    //
    // Ops[0] : target
    // Ops[1] : decoration
    // Ops[2] : SpecId
    SPIRVOperandVec Ops;
    Ops << RID << spv::DecorationBuiltIn << static_cast<int>(BID);

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  }

  return RID;
}

SPIRVID
SPIRVProducerPassImpl::GenerateClspvInstruction(CallInst *Call,
                                                const FunctionInfo &FuncInfo) {
  SPIRVID RID;

  switch (FuncInfo.getType()) {
  case Builtins::kClspvCompositeConstruct:
    RID = addSPIRVPlaceholder(Call);
    break;
  case Builtins::kClspvResource: {
    if (ResourceVarDeferredLoadCalls.count(Call) && Call->hasNUsesOrMore(1)) {
      // Generate an OpLoad
      SPIRVOperandVec Ops;

      Ops << Call->getArgOperand(ClspvOperand::kResourceDataType)->getType()
          << ResourceVarDeferredLoadCalls[Call];

      RID = addSPIRVInst(spv::OpLoad, Ops);

    } else {
      // This maps to an OpVariable we've already generated.
      // No code is generated for the call.
    }
    break;
  }
  case Builtins::kClspvLocal: {
    // Don't codegen an instruction here, but instead map this call directly
    // to the workgroup variable id.
    int spec_id = static_cast<int>(
        cast<ConstantInt>(Call->getOperand(ClspvOperand::kWorkgroupSpecId))
            ->getSExtValue());
    const auto &info = LocalSpecIdInfoMap[spec_id];
    RID = info.variable_id;
    break;
  }
  case Builtins::kClspvSamplerVarLiteral: {
    // Sampler initializers become a load of the corresponding sampler.
    // Map this to a load from the variable.
    const auto sampler_value = static_cast<unsigned>(
        dyn_cast<ConstantInt>(Call->getArgOperand(ClspvOperand::kSamplerParams))
            ->getZExtValue());

    // Generate an OpLoad
    SPIRVOperandVec Ops;

    Ops << SamplerDataTy << SamplerLiteralToIDMap[sampler_value];

    RID = addSPIRVInst(spv::OpLoad, Ops);
    break;
  }
  case Builtins::kSpirvAtomicXor: {
    // Handle SPIR-V intrinsics
    SPIRVOperandVec Ops;

    if (!Call->getType()->isVoidTy()) {
      Ops << Call->getType();
    }

    for (unsigned i = 0; i < Call->arg_size(); i++) {
      Ops << Call->getArgOperand(i);
    }

    RID = addSPIRVInst(spv::OpAtomicXor, Ops);
    break;
  }
  case Builtins::kSpirvOp: {
    // Handle SPIR-V intrinsics
    auto *arg0 = dyn_cast<ConstantInt>(Call->getArgOperand(0));
    spv::Op opcode = static_cast<spv::Op>(arg0->getZExtValue());
    if (opcode != spv::OpNop) {
      SPIRVOperandVec Ops;

      if (!Call->getType()->isVoidTy()) {
        Ops << Call->getType();
      }

      for (unsigned i = 1; i < Call->arg_size(); i++) {
        Ops << Call->getArgOperand(i);
      }

      RID = addSPIRVInst(opcode, Ops);
    }
    break;
  }
  case Builtins::kClspvGetNormalizedSamplerMask: {
    auto GV = module->getGlobalVariable(clspv::PushConstantsVariableName());
    auto offset = getSPIRVValue(mdconst::extract<ConstantInt>(
        Call->getMetadata(clspv::SamplerMaskPushConstantOffsetName())
            ->getOperand(0)));
    auto i32 = IntegerType::get(module->getContext(), 32);

    SPIRVOperandVec Ops;
    Ops << getSPIRVPointerType(
               PointerType::get(module->getContext(),
                                GV->getType()->getPointerAddressSpace()),
               i32)
        << getSPIRVValue(GV)
        << getSPIRVInt32Constant(GV->getValueType()->getStructNumElements() - 1)
        << offset;
    RID = addSPIRVInst(spv::OpAccessChain, Ops);

    Ops.clear();
    Ops << getSPIRVType(i32) << RID;
    RID = addSPIRVInst(spv::OpLoad, Ops);
    break;
  }
  default:
    llvm_unreachable("Unknown CLSPV Instruction");
    break;
  }
  return RID;
}

SPIRVID
SPIRVProducerPassImpl::GenerateImageInstruction(CallInst *Call,
                                                const FunctionInfo &FuncInfo) {
  SPIRVID RID;

  auto GetExtendMask = [](Type *sample_type, bool is_int_image) -> uint32_t {
    if (SpvVersion() >= SPIRVVersion::SPIRV_1_4 &&
        sample_type->getScalarType()->isIntegerTy()) {
      if (is_int_image)
        return spv::ImageOperandsSignExtendMask;
      else
        return spv::ImageOperandsZeroExtendMask;
    }
    return 0;
  };

  auto *image_ty = InferType(Call->getArgOperand(0), module->getContext(),
                             &InferredTypeCache);
  LLVMContext &Context = module->getContext();
  switch (FuncInfo.getType()) {
  case Builtins::kReadImagef:
  case Builtins::kReadImageh:
  case Builtins::kReadImagei:
  case Builtins::kReadImageui: {
    // read_image is converted to OpSampledImage and OpImageSampleExplicitLod.
    // Additionally, OpTypeSampledImage is generated.
    const auto &pi = FuncInfo.getParameter(1);
    if (pi.isSampler()) {
      //
      // Generate OpSampledImage.
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Image ID
      // Ops[2] = Sampler ID
      //
      SPIRVOperandVec Ops;

      Value *Image = Call->getArgOperand(0);
      Value *Sampler = Call->getArgOperand(1);
      Value *Coordinate = Call->getArgOperand(2);

      TypeMapType &OpImageTypeMap = getImageTypeMap();
      SPIRVID ImageTyID = OpImageTypeMap[image_ty];

      Ops << ImageTyID << Image << Sampler;

      SPIRVID SampledImageID = addSPIRVInst(spv::OpSampledImage, Ops);

      //
      // Generate OpImageSampleExplicitLod.
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Sampled Image ID
      // Ops[2] = Coordinate ID
      // Ops[3] = Image Operands Type ID
      // Ops[4] ... Ops[n] = Operands ID
      //
      Ops.clear();

      const bool is_int_image = IsIntImageType(image_ty);
      SPIRVID result_type;
      if (is_int_image) {
        result_type = v4int32ID;
      } else {
        result_type = getSPIRVType(Call->getType());
      }

      uint32_t mask = spv::ImageOperandsLodMask |
                      GetExtendMask(Call->getType(), is_int_image);
      Constant *CstFP0 = ConstantFP::get(Context, APFloat(0.0f));
      Ops << result_type << SampledImageID << Coordinate << mask << CstFP0;

      RID = addSPIRVInst(spv::OpImageSampleExplicitLod, Ops);

      if (is_int_image) {
        // Generate the bitcast.
        Ops.clear();
        Ops << Call->getType() << RID;
        RID = addSPIRVInst(spv::OpBitcast, Ops);
      }
    } else if (IsStorageImageType(image_ty)) {
      // read_image on a storage image is mapped to OpImageRead.
      Value *Image = Call->getArgOperand(0);
      Value *Coordinate = Call->getArgOperand(1);

      //
      // Generate OpImageRead
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Image ID
      // Ops[2] = Coordinate
      // No optional image operands.
      //
      SPIRVOperandVec Ops;

      const bool is_int_image = IsIntImageType(image_ty);
      SPIRVID result_type;
      if (is_int_image) {
        result_type = v4int32ID;
      } else {
        result_type = getSPIRVType(Call->getType());
      }

      Ops << result_type << Image << Coordinate;
      uint32_t mask = GetExtendMask(Call->getType(), is_int_image);
      if (mask != 0)
        Ops << mask;
      RID = addSPIRVInst(spv::OpImageRead, Ops);

      if (is_int_image) {
        // Generate the bitcast.
        Ops.clear();
        Ops << Call->getType() << RID;
        RID = addSPIRVInst(spv::OpBitcast, Ops);
      }

      // OpImageRead requires StorageImageReadWithoutFormat.
      addCapability(spv::CapabilityStorageImageReadWithoutFormat);
    } else {
      // read_image on a sampled image (without a sampler) is mapped to
      // OpImageFetch.
      Value *Image = Call->getArgOperand(0);
      Value *Coordinate = Call->getArgOperand(1);

      //
      // Generate OpImageFetch
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Image ID
      // Ops[2] = Coordinate ID
      // Ops[3] = Lod
      // Ops[4] = 0
      //
      SPIRVOperandVec Ops;

      const bool is_int_image = IsIntImageType(image_ty);
      SPIRVID result_type;
      if (is_int_image) {
        result_type = v4int32ID;
      } else {
        result_type = getSPIRVType(Call->getType());
      }
      Ops << result_type << Image << Coordinate;
      if (ImageDimensionality(image_ty) != spv::DimBuffer) {
        uint32_t mask = spv::ImageOperandsLodMask |
                        GetExtendMask(Call->getType(), is_int_image);
        Ops << mask << getSPIRVInt32Constant(0);
      }

      RID = addSPIRVInst(spv::OpImageFetch, Ops);

      if (is_int_image) {
        // Generate the bitcast.
        Ops.clear();
        Ops << Call->getType() << RID;
        RID = addSPIRVInst(spv::OpBitcast, Ops);
      }
    }
    break;
  }

  case Builtins::kWriteImagef:
  case Builtins::kWriteImageh:
  case Builtins::kWriteImagei:
  case Builtins::kWriteImageui: {
    // write_image is mapped to OpImageWrite.
    //
    // Generate OpImageWrite.
    //
    // Ops[0] = Image ID
    // Ops[1] = Coordinate ID
    // Ops[2] = Texel ID
    // Ops[3] = (Optional) Image Operands Type (Literal Number)
    // Ops[4] ... Ops[n] = (Optional) Operands ID
    //
    SPIRVOperandVec Ops;

    Value *Image = Call->getArgOperand(0);
    Value *Coordinate = Call->getArgOperand(1);
    Value *Texel = Call->getArgOperand(2);

    SPIRVID TexelID = getSPIRVValue(Texel);

    const bool is_int_image = IsIntImageType(image_ty);
    if (is_int_image) {
      // Generate a bitcast to v4int and use it as the texel value.
      Ops << v4int32ID << TexelID;
      TexelID = addSPIRVInst(spv::OpBitcast, Ops);
      Ops.clear();
    }
    Ops << Image << Coordinate << TexelID;
    uint32_t mask = GetExtendMask(Texel->getType(), is_int_image);
    if (mask != 0)
      Ops << mask;
    RID = addSPIRVInst(spv::OpImageWrite, Ops);

    // Image writes require StorageImageWriteWithoutFormat.
    addCapability(spv::CapabilityStorageImageWriteWithoutFormat);
    break;
  }

  case Builtins::kGetImageHeight:
  case Builtins::kGetImageWidth:
  case Builtins::kGetImageDepth:
  case Builtins::kGetImageArraySize:
  case Builtins::kGetImageDim: {
    // get_image_* is mapped to OpImageQuerySize or OpImageQuerySizeLod
    addCapability(spv::CapabilityImageQuery);

    //
    // Generate OpImageQuerySize[Lod]
    //
    // Ops[0] = Image ID
    //
    // Result type has components equal to the dimensionality of the image,
    // plus 1 if the image is arrayed.
    //
    // %sizes = OpImageQuerySize[Lod] %uint[2|3|4] %im [%uint_0]
    SPIRVOperandVec Ops;

    // Implement:
    //     %sizes = OpImageQuerySize[Lod] %uint[2|3|4] %im [%uint_0]
    SPIRVID SizesTypeID;

    Value *Image = Call->getArgOperand(0);
    const uint32_t dim = ImageNumDimensions(image_ty);
    const uint32_t components =
        dim + (IsArrayImageType(image_ty) ? 1 : 0);
    if (components == 1) {
      SizesTypeID = getSPIRVType(Type::getInt32Ty(Context));
    } else {
      SizesTypeID = getSPIRVType(
          FixedVectorType::get(Type::getInt32Ty(Context), components));
    }
    Ops << SizesTypeID << Image;
    spv::Op query_opcode = spv::OpImageQuerySize;
    if (IsSampledImageType(image_ty) &&
        ImageDimensionality(image_ty) != spv::DimBuffer) {
      query_opcode = spv::OpImageQuerySizeLod;
      // Need explicit 0 for Lod operand.
      Ops << getSPIRVInt32Constant(0);
    }

    RID = addSPIRVInst(query_opcode, Ops);

    // May require an extra instruction to create the appropriate result of
    // the builtin function.
    if (FuncInfo.getType() == Builtins::kGetImageDim) {
      if (dim == 3) {
        // get_image_dim returns an int4 for 3D images.
        //

        // Implement:
        //   %result = OpCompositeConstruct %uint4 %sizes %uint_0
        Ops.clear();
        Ops << FixedVectorType::get(Type::getInt32Ty(Context), 4) << RID
            << getSPIRVInt32Constant(0);

        RID = addSPIRVInst(spv::OpCompositeConstruct, Ops);
      } else if (dim != components) {
        // get_image_dim return an int2 regardless of the arrayedness of the
        // image. If the image is arrayed an element must be dropped from the
        // query result.
        //

        // Implement:
        //   %result = OpVectorShuffle %uint2 %sizes %sizes 0 1
        Ops.clear();
        Ops << FixedVectorType::get(Type::getInt32Ty(Context), 2) << RID << RID
            << 0 << 1;

        RID = addSPIRVInst(spv::OpVectorShuffle, Ops);
      }
    } else if (components > 1) {
      // Implement:
      //     %result = OpCompositeExtract %uint %sizes <component number>
      Ops.clear();
      Ops << Type::getInt32Ty(Context) << RID;

      uint32_t component = 0;
      if (FuncInfo.getType() == Builtins::kGetImageHeight)
        component = 1;
      else if (FuncInfo.getType() == Builtins::kGetImageDepth)
        component = 2;
      else if (FuncInfo.getType() == Builtins::kGetImageArraySize)
        component =
            components - 1; // array size is the last element of the vector
      Ops << component;

      RID = addSPIRVInst(spv::OpCompositeExtract, Ops);

      // get_image_array_size returns a size_t so we need to extend it to an
      // i64 if size_t is 64-bit
      if (FuncInfo.getType() == Builtins::kGetImageArraySize &&
          clspv::PointersAre64Bit(*module)) {
        Ops.clear();
        Ops << Type::getInt64Ty(Context) << RID;
        RID = addSPIRVInst(spv::OpUConvert, Ops);
      }
    }
    break;
  }
  case kGetImageChannelDataType:
  case kGetImageChannelOrder: {
    auto GV = module->getGlobalVariable(clspv::PushConstantsVariableName());
    auto offset = getSPIRVValue(mdconst::extract<ConstantInt>(
        Call->getMetadata(clspv::ImageGetterPushConstantOffsetName())
            ->getOperand(0)));
    auto i32 = IntegerType::get(module->getContext(), 32);

    SPIRVOperandVec Ops;
    Ops << getSPIRVPointerType(
               PointerType::get(module->getContext(),
                                GV->getType()->getPointerAddressSpace()),
               i32)
        << getSPIRVValue(GV)
        << getSPIRVInt32Constant(GV->getValueType()->getStructNumElements() - 1)
        << offset;
    RID = addSPIRVInst(spv::OpAccessChain, Ops);

    Ops.clear();
    Ops << getSPIRVType(i32) << RID;
    RID = addSPIRVInst(spv::OpLoad, Ops);
  } break;
  default:
    llvm_unreachable("Unsupported Image builtin");
  }

  return RID;
}

SPIRVID
SPIRVProducerPassImpl::GenerateSubgroupInstruction(
    CallInst *Call, const FunctionInfo &FuncInfo) {

  // requires SPIRV version 1.3 or greater
  if (SpvVersion() != SPIRVVersion::SPIRV_1_3) {
    // llvm_unreachable("SubGroups extension requires SPIRV 1.3 or greater");
    // TODO(sjw): error out gracefully
  }

  auto loadBuiltin = [this, Call](spv::BuiltIn spvBI,
                                  spv::Capability spvCap =
                                      spv::CapabilityGroupNonUniform) {
    SPIRVOperandVec Ops;
    Ops << Call->getType()
        << this->getSPIRVBuiltin(spvBI, Call->getType(), spvCap);

    return addSPIRVInst(spv::OpLoad, Ops);
  };

  spv::Op op = spv::OpNop;
  switch (FuncInfo.getType()) {
  case Builtins::kGetSubGroupSize: {
    // To support partial subgroups correctly (when WorkgroupSize is not a
    // multiple of SubgroupSize), OpenCL expects the true remaining invocations
    // for the last subgroup. Since Vulkan's spv::BuiltInSubgroupSize always
    // returns the max hardware warp size, we compute this manually using
    // FlatWorkgroupSize.
    auto Int32Ty = Type::getInt32Ty(module->getContext());
    assert(WorkgroupSizeValueID.isValid());

    // 2. Extract WGSize X, Y, Z
    SPIRVOperandVec Ops;
    Ops << Int32Ty << WorkgroupSizeValueID << 0;
    SPIRVID WGSizeX = addSPIRVInst(spv::OpCompositeExtract, Ops);

    Ops.clear();
    Ops << Int32Ty << WorkgroupSizeValueID << 1;
    SPIRVID WGSizeY = addSPIRVInst(spv::OpCompositeExtract, Ops);

    Ops.clear();
    Ops << Int32Ty << WorkgroupSizeValueID << 2;
    SPIRVID WGSizeZ = addSPIRVInst(spv::OpCompositeExtract, Ops);

    // 3. Multiply X, Y, Z to get FlatWorkgroupSize
    Ops.clear();
    Ops << Int32Ty << WGSizeX << WGSizeY;
    SPIRVID FlatWGSizeXY = addSPIRVInst(spv::OpIMul, Ops);

    Ops.clear();
    Ops << Int32Ty << FlatWGSizeXY << WGSizeZ;
    SPIRVID FlatWGSize = addSPIRVInst(spv::OpIMul, Ops);

    // 4. Load SubgroupId and MaxSubgroupSize
    SPIRVID SubgroupId = loadBuiltin(spv::BuiltInSubgroupId);
    SPIRVID MaxSubgroupSize = loadBuiltin(spv::BuiltInSubgroupSize);

    // 5. SubgroupStartId = SubgroupId * MaxSubgroupSize
    Ops.clear();
    Ops << Int32Ty << SubgroupId << MaxSubgroupSize;
    SPIRVID SubgroupStartId = addSPIRVInst(spv::OpIMul, Ops);

    // 6. Remaining = FlatWorkgroupSize - SubgroupStartId
    Ops.clear();
    Ops << Int32Ty << FlatWGSize << SubgroupStartId;
    SPIRVID Remaining = addSPIRVInst(spv::OpISub, Ops);

    // 7. ActualSize = ExtInstUMin(Remaining, MaxSubgroupSize)
    Ops.clear();
    Ops << Call->getType() << getOpExtInstImportID()
        << glsl::ExtInst::ExtInstUMin << Remaining << MaxSubgroupSize;
    return addSPIRVInst(spv::OpExtInst, Ops);
  }
  case Builtins::kGetNumSubGroups:
    return loadBuiltin(spv::BuiltInNumSubgroups);
  case Builtins::kGetSubGroupId:
    return loadBuiltin(spv::BuiltInSubgroupId);
  case Builtins::kGetSubGroupLocalId:
    return loadBuiltin(spv::BuiltInSubgroupLocalInvocationId);

  case Builtins::kSubGroupBroadcast:
  case Builtins::kSubGroupNonUniformBroadcast:
    if (SpvVersion() < SPIRVVersion::SPIRV_1_5 &&
        !dyn_cast<ConstantInt>(Call->getOperand(1))) {
      llvm_unreachable("sub_group_broadcast requires constant lane Id for "
                       "SPIRV version < 1.5");
    }
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBroadcast;
    break;
  case Builtins::kSubGroupBroadcastFirst:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBroadcastFirst;
    break;

  case Builtins::kSubGroupAll:
    addCapability(spv::CapabilityGroupNonUniformVote);
    op = spv::OpGroupNonUniformAll;
    break;
  case Builtins::kSubGroupAny:
    addCapability(spv::CapabilityGroupNonUniformVote);
    op = spv::OpGroupNonUniformAny;
    break;
  case Builtins::kSubGroupBallot:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBallot;
    break;
  case Builtins::kSubGroupBallotFindLSB:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBallotFindLSB;
    break;
  case Builtins::kSubGroupBallotFindMSB:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBallotFindMSB;
    break;
  case Builtins::kSubGroupAllEqual:
    addCapability(spv::CapabilityGroupNonUniformVote);
    op = spv::OpGroupNonUniformAllEqual;
    break;
  case Builtins::kSubGroupInverseBallot:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformInverseBallot;
    break;
  case Builtins::kSubGroupBallotBitExtract:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBallotBitExtract;
    break;
  case Builtins::kSubGroupBallotBitCount:
  case Builtins::kSubGroupBallotInclusiveScan:
  case Builtins::kSubGroupBallotExclusiveScan:
    addCapability(spv::CapabilityGroupNonUniformBallot);
    op = spv::OpGroupNonUniformBallotBitCount;
    break;
  case Builtins::kSubGroupElect:
    addCapability(spv::CapabilityGroupNonUniformVote);
    op = spv::OpGroupNonUniformElect;
    break;
  case Builtins::kGetSubGroupEqMask:
    return loadBuiltin(spv::BuiltInSubgroupEqMask,
                       spv::CapabilityGroupNonUniformBallot);
  case Builtins::kGetSubGroupGeMask:
    return loadBuiltin(spv::BuiltInSubgroupGeMask,
                       spv::CapabilityGroupNonUniformBallot);
  case Builtins::kGetSubGroupGtMask:
    return loadBuiltin(spv::BuiltInSubgroupGtMask,
                       spv::CapabilityGroupNonUniformBallot);
  case Builtins::kGetSubGroupLeMask:
    return loadBuiltin(spv::BuiltInSubgroupLeMask,
                       spv::CapabilityGroupNonUniformBallot);
  case Builtins::kGetSubGroupLtMask:
    return loadBuiltin(spv::BuiltInSubgroupLtMask,
                       spv::CapabilityGroupNonUniformBallot);
  case Builtins::kSubGroupReduceAdd:
  case Builtins::kSubGroupScanExclusiveAdd:
  case Builtins::kSubGroupScanInclusiveAdd: {
    addCapability(spv::CapabilityGroupNonUniformArithmetic);
    if (FuncInfo.getParameter(0).type_id == Type::IntegerTyID) {
      op = spv::OpGroupNonUniformIAdd;
    } else {
      op = spv::OpGroupNonUniformFAdd;
    }
    break;
  }
  case Builtins::kSubGroupReduceMin:
  case Builtins::kSubGroupScanExclusiveMin:
  case Builtins::kSubGroupScanInclusiveMin: {
    addCapability(spv::CapabilityGroupNonUniformArithmetic);
    auto &param = FuncInfo.getParameter(0);
    if (param.type_id == Type::IntegerTyID) {
      op = param.is_signed ? spv::OpGroupNonUniformSMin
                           : spv::OpGroupNonUniformUMin;
    } else {
      op = spv::OpGroupNonUniformFMin;
    }
    break;
  }
  case Builtins::kSubGroupReduceMax:
  case Builtins::kSubGroupScanExclusiveMax:
  case Builtins::kSubGroupScanInclusiveMax: {
    addCapability(spv::CapabilityGroupNonUniformArithmetic);
    auto &param = FuncInfo.getParameter(0);
    if (param.type_id == Type::IntegerTyID) {
      op = param.is_signed ? spv::OpGroupNonUniformSMax
                           : spv::OpGroupNonUniformUMax;
    } else {
      op = spv::OpGroupNonUniformFMax;
    }
    break;
  }
  case Builtins::kSubGroupShuffle:
    addCapability(spv::CapabilityGroupNonUniformShuffle);
    op = spv::OpGroupNonUniformShuffle;
    break;
  case Builtins::kSubGroupShuffleXor:
    addCapability(spv::CapabilityGroupNonUniformShuffle);
    op = spv::OpGroupNonUniformShuffleXor;
    break;
  case Builtins::kSubGroupRotate:
    addCapability(spv::CapabilityGroupNonUniformRotateKHR);
    // OpGroupNonUniformRotateKHR is defined by the SPV_KHR_subgroup_rotate
    // extension,thus it needs to be explicitly enabled.
    addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_subgroup_rotate");
    op = spv::OpGroupNonUniformRotateKHR;
    break;

  case Builtins::kGetEnqueuedNumSubGroups:
  case Builtins::kGetMaxSubGroupSize:
  case Builtins::kSubGroupBarrier:
  case Builtins::kSubGroupReserveReadPipe:
  case Builtins::kSubGroupReserveWritePipe:
  case Builtins::kSubGroupCommitReadPipe:
  case Builtins::kSubGroupCommitWritePipe:
  case Builtins::kGetKernelSubGroupCountForNdrange:
  case Builtins::kGetKernelMaxSubGroupSizeForNdrange:
  default:
    Call->print(errs());
    llvm_unreachable("Unsupported sub_group operation");
    break;
  }

  assert(op != spv::OpNop);

  SPIRVOperandVec Operands;

  //
  // Generate OpGroupNonUniform*
  //
  // Ops[0] = Result Type ID
  // Ops[1] = ScopeSubgroup
  // Ops[2] = Value ID
  // Ops[3] = Local ID

  // The result type.
  switch (FuncInfo.getType()) {
  case Builtins::kSubGroupAny:
  case Builtins::kSubGroupAll:
  case Builtins::kSubGroupElect:
  case Builtins::kSubGroupAllEqual:
  case Builtins::kSubGroupInverseBallot:
  case Builtins::kSubGroupBallotBitExtract:
    // SPIR-V needs a bool return type for any/all/elect/all_equal.
    Operands << getSPIRVType(Type::getInt1Ty(module->getContext()));
    break;
  default:
    Operands << Call->getType();
    break;
  }

  // Subgroup Scope
  Operands << getSPIRVInt32Constant(spv::ScopeSubgroup);

  switch (FuncInfo.getType()) {
  case Builtins::kSubGroupReduceAdd:
  case Builtins::kSubGroupReduceMin:
  case Builtins::kSubGroupReduceMax:
  case Builtins::kSubGroupBallotBitCount:
    Operands << spv::GroupOperationReduce;
    break;
  case Builtins::kSubGroupScanExclusiveAdd:
  case Builtins::kSubGroupScanExclusiveMin:
  case Builtins::kSubGroupScanExclusiveMax:
  case Builtins::kSubGroupBallotExclusiveScan:
    Operands << spv::GroupOperationExclusiveScan;
    break;
  case Builtins::kSubGroupScanInclusiveAdd:
  case Builtins::kSubGroupScanInclusiveMin:
  case Builtins::kSubGroupScanInclusiveMax:
  case Builtins::kSubGroupBallotInclusiveScan:
    Operands << spv::GroupOperationInclusiveScan;
    break;
  default:
    break;
  }

  SPIRVID RID;
  switch (FuncInfo.getType()) {
  case Builtins::kSubGroupAny:
  case Builtins::kSubGroupAll: {
    // OpenCL uses an int predicate instead of the bool from SPIR-V.
    // Generate:
    // tmp = predicate != 0
    // op = Any/All predicate
    // RID = op ? 1 : 0
    SPIRVOperandVec Ops;
    Ops << getSPIRVType(Type::getInt1Ty(module->getContext()))
        << Call->getArgOperand(0) << ConstantInt::get(Call->getType(), 0);
    auto cmp = addSPIRVInst(spv::OpINotEqual, Ops);
    Ops.clear();
    Operands << cmp;
    auto subgroup = addSPIRVInst(op, Operands);
    Ops << Call->getType() << subgroup << ConstantInt::get(Call->getType(), 1)
        << ConstantInt::get(Call->getType(), 0);
    RID = addSPIRVInst(spv::OpSelect, Ops);
    break;
  }
  case Builtins::kSubGroupAllEqual: {
    Operands << Call->getArgOperand(0);
    auto subgroup = addSPIRVInst(op, Operands);
    SPIRVOperandVec Ops;
    Ops << Call->getType() << subgroup << ConstantInt::get(Call->getType(), 1)
        << ConstantInt::get(Call->getType(), 0);
    RID = addSPIRVInst(spv::OpSelect, Ops);
    break;
  }
  case Builtins::kSubGroupInverseBallot: {
    Operands << Call->getArgOperand(0);
    auto subgroup = addSPIRVInst(op, Operands);
    SPIRVOperandVec Ops;
    Ops << Call->getType() << subgroup << ConstantInt::get(Call->getType(), 1)
        << ConstantInt::get(Call->getType(), 0);
    RID = addSPIRVInst(spv::OpSelect, Ops);
    break;
  }
  case Builtins::kSubGroupBallotBitExtract: {
    Operands << Call->getArgOperand(0) << Call->getArgOperand(1);
    auto subgroup = addSPIRVInst(op, Operands);
    SPIRVOperandVec Ops;
    Ops << Call->getType() << subgroup << ConstantInt::get(Call->getType(), 1)
        << ConstantInt::get(Call->getType(), 0);
    RID = addSPIRVInst(spv::OpSelect, Ops);
    break;
  }
  case Builtins::kSubGroupBallot: {
    SPIRVOperandVec Ops;
    Ops << getSPIRVType(Type::getInt1Ty(module->getContext()))
        << Call->getArgOperand(0)
        << ConstantInt::get(Call->getArgOperand(0)->getType(), 0);
    auto cmp = addSPIRVInst(spv::OpINotEqual, Ops);
    Ops.clear();
    Operands << cmp;
    RID = addSPIRVInst(op, Operands);
    break;
  }
  case Builtins::kSubGroupElect: {
    auto Int1Ty = Type::getInt1Ty(module->getContext());
    auto Int32Ty = Type::getInt32Ty(module->getContext());

    SPIRVOperandVec Ops;
    Ops << getSPIRVType(Int1Ty)
        << ConstantInt::get(Int32Ty, spv::ScopeSubgroup);

    RID = addSPIRVInst(spv::OpGroupNonUniformElect, Ops);

    SPIRVOperandVec SelectOps;
    SelectOps << getSPIRVType(Int32Ty) << RID << ConstantInt::get(Int32Ty, 1)
              << ConstantInt::get(Int32Ty, 0);

    RID = addSPIRVInst(spv::OpSelect, SelectOps);
    break;
  }
  default:
    for (Use &use : Call->args()) {
      Operands << use.get();
    }
    RID = addSPIRVInst(op, Operands);
    break;
  }

  return RID;
}

SPIRVID SPIRVProducerPassImpl::GenerateShuffle2FromCall(Type *Ty, Value *SrcA,
                                                        Value *SrcB,
                                                        Value *Mask) {
  assert(Ty->isVectorTy());
  assert(SrcA->getType() == SrcB->getType());
  auto MaskTy = Mask->getType();
  assert(MaskTy->isVectorTy());
  auto MaskVTy = cast<FixedVectorType>(MaskTy);
  auto MaskNumElements = MaskVTy->getNumElements();
  SPIRVID RID;

  if (auto CstMask = dyn_cast<ConstantDataVector>(Mask)) {
    SPIRVOperandVec Ops;
    Ops << Ty << SrcA << SrcB;
    for (unsigned int each = 0; each < MaskNumElements; each++) {
      auto CstInt = dyn_cast<ConstantInt>(CstMask->getAggregateElement(each));
      assert(CstInt);
      Ops << (unsigned)CstInt->getZExtValue();
    }
    RID = addSPIRVInst(spv::OpVectorShuffle, Ops);
  } else {
    auto VTy = cast<FixedVectorType>(Ty);
    auto ScalarTy = VTy->getScalarType();
    auto MaskScalarTy = MaskVTy->getScalarType();
    auto NumElements =
        (cast<FixedVectorType>(SrcA->getType()))->getNumElements();

    SPIRVOperandVec Ops;
    Ops << VTy;
    RID = addSPIRVInst(spv::OpUndef, Ops);

    bool isShuffle2 = SrcA != SrcB;

    auto CstNumElements = getSPIRVValue(
        ConstantInt::get(Type::getIntNTy(module->getContext(),
                                         MaskScalarTy->getScalarSizeInBits()),
                         NumElements));

    for (unsigned each = 0; each < MaskNumElements; each++) {
      Ops.clear();
      Ops << MaskScalarTy << Mask << each;
      auto Index = addSPIRVInst(spv::OpCompositeExtract, Ops);

      Ops.clear();
      Ops << MaskScalarTy << Index << CstNumElements;
      auto IndexMod = addSPIRVInst(spv::OpUMod, Ops);

      Ops.clear();
      Ops << ScalarTy << SrcA << IndexMod;
      auto Value = addSPIRVInst(spv::OpVectorExtractDynamic, Ops);

      if (isShuffle2) {
        Ops.clear();
        Ops << ScalarTy << SrcB << IndexMod;
        auto ValueB = addSPIRVInst(spv::OpVectorExtractDynamic, Ops);

        Ops.clear();
        Ops << MaskScalarTy << Index
            << getSPIRVValue(ConstantInt::get(
                   Type::getIntNTy(module->getContext(),
                                   MaskScalarTy->getScalarSizeInBits()),
                   NumElements * 2));
        auto IndexMod2 = addSPIRVInst(spv::OpUMod, Ops);

        Ops.clear();
        Ops << Type::getInt1Ty(module->getContext()) << IndexMod2
            << CstNumElements;
        auto Cmp = addSPIRVInst(spv::OpUGreaterThanEqual, Ops);

        Ops.clear();
        Ops << ScalarTy << Cmp << ValueB << Value;
        Value = addSPIRVInst(spv::OpSelect, Ops);
      }

      Ops.clear();
      Ops << VTy << Value << RID << each;
      RID = addSPIRVInst(spv::OpCompositeInsert, Ops);
    }
  }
  return RID;
}

SPIRVID SPIRVProducerPassImpl::GeneratePopcount(Type *Ty, Value *BaseValue,
                                                LLVMContext &Context) {
  // The Base operand of any OpBitCount, OpBitReverse, OpBitFieldInsert,
  // OpBitFieldSExtract, or OpBitFieldUExtract instruction must be a 32-bit
  // integer scalar or a vector of 32-bit integers.
  // https://www.khronos.org/registry/vulkan/specs/1.3-extensions/html/chap46.html#VUID-StandaloneSpirv-Base-04781
  // Thus, non 32-bit types need to be managed explicitly.
  SPIRVID RID;
  const int BitcountSize = 32;
  Type *Int32Ty = Ty->getInt32Ty(Context);
  if (Ty->isVectorTy())
    Int32Ty = VectorType::get(Int32Ty, dyn_cast<VectorType>(Ty));
  auto TyBitWidth = Ty->getScalarSizeInBits();

  SPIRVOperandVec Ops;
  if (TyBitWidth == BitcountSize * 2) {
    Ops.clear();
    Ops << Int32Ty << BaseValue;
    auto convertLower = addSPIRVInst(spv::OpUConvert, Ops);

    Ops.clear();
    Ops << Int32Ty << convertLower;
    auto bitcountLower = addSPIRVInst(spv::OpBitCount, Ops);

    Ops.clear();
    Ops << Ty << BaseValue << ConstantInt::get(Ty, BitcountSize);
    auto UpperValue = addSPIRVInst(spv::OpShiftRightLogical, Ops);

    Ops.clear();
    Ops << Int32Ty << UpperValue;
    auto convertUpper = addSPIRVInst(spv::OpUConvert, Ops);

    Ops.clear();
    Ops << Int32Ty << convertUpper;
    auto bitcountUpper = addSPIRVInst(spv::OpBitCount, Ops);

    Ops.clear();
    Ops << Int32Ty << bitcountLower << bitcountUpper;
    RID = addSPIRVInst(spv::OpIAdd, Ops);

    Ops.clear();
    Ops << Ty << RID;
    RID = addSPIRVInst(spv::OpUConvert, Ops);
  } else if (TyBitWidth == BitcountSize) {
    Ops << Ty << BaseValue;
    RID = addSPIRVInst(spv::OpBitCount, Ops);
  } else if (TyBitWidth < BitcountSize) {
    Ops << Int32Ty << BaseValue;
    RID = addSPIRVInst(spv::OpUConvert, Ops);

    Ops.clear();
    Ops << Int32Ty << RID;
    RID = addSPIRVInst(spv::OpBitCount, Ops);

    Ops.clear();
    Ops << Ty << RID;
    RID = addSPIRVInst(spv::OpUConvert, Ops);
  } else {
    llvm_unreachable("Unsupported type width for kpopcount");
  }
  return RID;
}

SPIRVID SPIRVProducerPassImpl::GenerateFabs(SPIRVID Input, Type *InputTy) {
  SPIRVOperandVec Ops;

  auto nativeBuiltins = clspv::Option::UseNativeBuiltins();
  if (nativeBuiltins.count(Builtins::kFabs)) {
    Ops << InputTy << getOpExtInstImportID() << glsl::ExtInst::ExtInstFAbs
        << Input;
    return addSPIRVInst(spv::OpExtInst, Ops);
  }

  auto getSameSizeIntegerTy = [](Type *FpTy) {
    return Type::getIntNTy(FpTy->getContext(), FpTy->getScalarSizeInBits());
  };
  Type *IntegerTy;
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    IntegerTy = FixedVectorType::get(
        getSameSizeIntegerTy(VecTy->getElementType()), VecTy->getNumElements());
  } else {
    IntegerTy = getSameSizeIntegerTy(InputTy);
  }
  Ops << IntegerTy << Input;
  auto bitcast = addSPIRVInst(spv::OpBitcast, Ops);
  Ops.clear();
  APInt abs_mask_val = ~APInt::getSignMask(InputTy->getScalarSizeInBits());
  Constant *Mask = ConstantInt::get(IntegerTy->getScalarType(), abs_mask_val);
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    Mask = ConstantVector::getSplat(VecTy->getElementCount(), Mask);
  }
  Ops << IntegerTy << bitcast << Mask;
  auto bitwiseand = addSPIRVInst(spv::OpBitwiseAnd, Ops);
  Ops.clear();
  Ops << InputTy << bitwiseand;
  return addSPIRVInst(spv::OpBitcast, Ops);
}

SPIRVID SPIRVProducerPassImpl::GenerateFabs(Value *Input) {
  return GenerateFabs(getSPIRVValue(Input), Input->getType());
}

bool SPIRVProducerPassImpl::ShouldFlushDenorms(Type *Ty) const {
  Type *ScalarTy = Ty->getScalarType();
  if (ScalarTy->isHalfTy()) {
    return clspv::Option::ExecutionModeDenorm(
               clspv::Option::FloatingPointType::fp16) ==
           clspv::Option::DenormMode::flush_to_zero;
  } else if (ScalarTy->isFloatTy()) {
    return clspv::Option::ExecutionModeDenorm(
               clspv::Option::FloatingPointType::fp32) !=
           clspv::Option::DenormMode::preserve;
  } else if (ScalarTy->isDoubleTy()) {
    return clspv::Option::ExecutionModeDenorm(
               clspv::Option::FloatingPointType::fp64) !=
           clspv::Option::DenormMode::preserve;
  }
  return false;
}

SPIRVID SPIRVProducerPassImpl::GenerateCopySignZero(SPIRVID val,
                                                    Type *InputTy) {
  auto getSameSizeIntegerTy = [](Type *FpTy) {
    return Type::getIntNTy(FpTy->getContext(), FpTy->getScalarSizeInBits());
  };
  Type *IntegerTy;
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    IntegerTy = FixedVectorType::get(
        getSameSizeIntegerTy(VecTy->getElementType()), VecTy->getNumElements());
  } else {
    IntegerTy = getSameSizeIntegerTy(InputTy);
  }

  SPIRVOperandVec Ops;
  Ops << IntegerTy << val;
  auto bitcast_val = addSPIRVInst(spv::OpBitcast, Ops);

  APInt sign_mask_val = APInt::getSignMask(InputTy->getScalarSizeInBits());
  Constant *SignMask =
      ConstantInt::get(IntegerTy->getScalarType(), sign_mask_val);
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    SignMask = ConstantVector::getSplat(VecTy->getElementCount(), SignMask);
  }

  Ops.clear();
  Ops << IntegerTy << bitcast_val << SignMask;
  auto copysign_zero_int = addSPIRVInst(spv::OpBitwiseAnd, Ops);

  Ops.clear();
  Ops << InputTy << copysign_zero_int;
  return addSPIRVInst(spv::OpBitcast, Ops);
}

SPIRVID SPIRVProducerPassImpl::GenerateCanonicalize(SPIRVID val,
                                                    Type *InputTy) {
  auto &Context = InputTy->getContext();
  auto copysign_zero = GenerateCopySignZero(val, InputTy);
  auto abs_val = GenerateFabs(val, InputTy);

  auto flt_min_ap = llvm::APFloat::getSmallestNormalized(
      InputTy->getScalarType()->getFltSemantics());
  Constant *flt_min_cnst =
      ConstantFP::get(InputTy->getScalarType(), flt_min_ap);
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    flt_min_cnst =
        ConstantVector::getSplat(VecTy->getElementCount(), flt_min_cnst);
  }

  Type *BoolTy = Type::getInt1Ty(Context);
  if (auto VecTy = dyn_cast<FixedVectorType>(InputTy)) {
    BoolTy = FixedVectorType::get(BoolTy, VecTy->getNumElements());
  }

  SPIRVOperandVec Ops;
  Ops << BoolTy << abs_val << flt_min_cnst;
  auto cmp = addSPIRVInst(spv::OpFOrdLessThan, Ops);

  Ops.clear();
  Ops << InputTy << cmp << copysign_zero << val;
  return addSPIRVInst(spv::OpSelect, Ops);
}

SPIRVID SPIRVProducerPassImpl::GenerateCanonicalize(Value *val) {
  return GenerateCanonicalize(getSPIRVValue(val), val->getType());
}

SPIRVID
SPIRVProducerPassImpl::GenerateIntegerDot(CallInst *Call,
                                          const FunctionInfo &func_info) {
  setIntegerDot();
  addCapability(spv::CapabilityDotProduct);
  auto a_val = Call->getOperand(0);
  auto b_val = Call->getOperand(1);
  assert(a_val->getType() == b_val->getType());
  auto input_ty = a_val->getType();
  auto a = getSPIRVValue(a_val);
  auto b = getSPIRVValue(b_val);
  auto IsIxJBit = [](Type *Ty, unsigned i, unsigned j) {
    auto VecTy = dyn_cast<FixedVectorType>(Ty);
    if (VecTy == nullptr) {
      return false;
    }
    if (VecTy->getNumElements() != i) {
      return false;
    }
    if (!VecTy->getScalarType()->isIntegerTy() ||
        VecTy->getScalarSizeInBits() != j) {
      return false;
    }
    return true;
  };
  bool Is4x8Bit = IsIxJBit(input_ty, 4, 8);
  bool Is2x16Bit = IsIxJBit(input_ty, 2, 16);
  bool Is1x32Bit = input_ty->isIntegerTy() && !input_ty->isVectorTy() &&
                   input_ty->getScalarSizeInBits() == 32;
  if (Is4x8Bit && clspv::Option::Int8Support()) {
    addCapability(spv::CapabilityDotProductInput4x8Bit);
  }
  if (Is2x16Bit) {
    addCapability(spv::CapabilityDotProductInputAll);
  }
  if (Is1x32Bit || (!clspv::Option::Int8Support() && Is4x8Bit)) {
    addCapability(spv::CapabilityDotProductInput4x8BitPacked);
  }

  SPIRVOperandVec Ops;
  switch (func_info.getType()) {
  case Builtins::kIDotAccSatPackedUUU: {
    Ops << Call->getType() << a << b << Call->getOperand(2)
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpUDotAccSat, Ops);
  }
  case Builtins::kIDotAccSatPackedSSS: {
    Ops << Call->getType() << a << b << Call->getOperand(2)
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSDotAccSat, Ops);
  }
  case Builtins::kIDotAccSatPackedSUS: {
    Ops << Call->getType() << a << b << Call->getOperand(2)
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSUDotAccSat, Ops);
  }
  case Builtins::kIDotAccSatPackedUSS: {
    Ops << Call->getType() << b << a << Call->getOperand(2)
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSUDotAccSat, Ops);
  }
  case Builtins::kIDotPackedUUU: {
    Ops << Call->getType() << a << b
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpUDot, Ops);
  }
  case Builtins::kIDotPackedSSS: {
    Ops << Call->getType() << a << b
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSDot, Ops);
  }
  case Builtins::kIDotPackedSUS: {
    Ops << Call->getType() << a << b
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSUDot, Ops);
  }
  case Builtins::kIDotPackedUSS: {
    Ops << Call->getType() << b << a
        << spv::PackedVectorFormat::PackedVectorFormatPackedVectorFormat4x8Bit;
    return addSPIRVInst(spv::OpSUDot, Ops);
  }
  case Builtins::kIDotAccSat: {
    spv::Op opcode;
    Ops << Call->getType();
    if (!func_info.getParameter(0).is_signed &&
        func_info.getParameter(1).is_signed) {
      opcode = spv::OpSUDotAccSat;
      Ops << b << a;
    } else {
      Ops << a << b;
      if (func_info.getParameter(0).is_signed &&
          func_info.getParameter(1).is_signed) {
        opcode = spv::OpSDotAccSat;
      } else if (!func_info.getParameter(0).is_signed &&
                 !func_info.getParameter(1).is_signed) {
        opcode = spv::OpUDotAccSat;
      } else {
        opcode = spv::OpSUDotAccSat;
      }
    }
    Ops << Call->getOperand(2);
    if (Is1x32Bit || !clspv::Option::Int8Support()) {
      Ops << spv::PackedVectorFormat::
              PackedVectorFormatPackedVectorFormat4x8BitKHR;
    }
    return addSPIRVInst(opcode, Ops);
  }
  case Builtins::kArmDotAcc: {
    spv::Op opcode;
    if (func_info.getParameter(0).is_signed) {
      opcode = spv::OpSDot;
    } else {
      opcode = spv::OpUDot;
    }
    Ops << Call->getType() << a << b;
    if (Is1x32Bit || (!clspv::Option::Int8Support() && !Is2x16Bit)) {
      Ops << spv::PackedVectorFormat::
              PackedVectorFormatPackedVectorFormat4x8BitKHR;
    }
    auto dot = addSPIRVInst(opcode, Ops);
    Ops.clear();

    Ops << Call->getType() << dot << Call->getOperand(2);
    return addSPIRVInst(spv::OpIAdd, Ops);
  }
  case Builtins::kDot: {
    spv::Op opcode;
    Ops << Call->getType();
    if (!func_info.getParameter(0).is_signed &&
        func_info.getParameter(1).is_signed) {
      opcode = spv::OpSUDot;
      Ops << b << a;
    } else {
      Ops << a << b;
      if (func_info.getParameter(0).is_signed &&
          func_info.getParameter(1).is_signed) {
        opcode = spv::OpSDot;
      } else if (!func_info.getParameter(0).is_signed &&
                 !func_info.getParameter(1).is_signed) {
        opcode = spv::OpUDot;
      } else {
        opcode = spv::OpSUDot;
      }
    }
    if (Is1x32Bit || !clspv::Option::Int8Support()) {
      Ops << spv::PackedVectorFormat::
              PackedVectorFormatPackedVectorFormat4x8BitKHR;
    }
    return addSPIRVInst(opcode, Ops);
  }
  default:
    llvm_unreachable("unexpected builtins in GenerateArmDot");
  }
}

SPIRVID SPIRVProducerPassImpl::GenerateInstructionFromCall(CallInst *Call) {
  LLVMContext &Context = module->getContext();

  auto &func_info = Builtins::Lookup(Call->getCalledFunction());
  auto func_type = func_info.getType();

  if (BUILTIN_IN_GROUP(func_type, Clspv)) {
    return GenerateClspvInstruction(Call, func_info);
  } else if (BUILTIN_IN_GROUP(func_type, Image)) {
    return GenerateImageInstruction(Call, func_info);
  } else if (BUILTIN_IN_GROUP(func_type, SubgroupsKHR)) {
    return GenerateSubgroupInstruction(Call, func_info);
  }

  SPIRVID RID;

  switch (Call->getCalledFunction()->getIntrinsicID()) {
  case Intrinsic::dbg_label:
  case Intrinsic::dbg_value:
  case Intrinsic::dbg_declare: {
    // TODO: Generate 'OpExtInst %void %<> Debug*'
    return RID;
  }
  case Intrinsic::ctlz: {
    // Implement as 31 - FindUMsb. Ignore the second operand of llvm.ctlz.
    SPIRVOperandVec Ops;
    Ops << Call->getType() << getOpExtInstImportID()
        << glsl::ExtInst::ExtInstFindUMsb << Call->getArgOperand(0);
    auto find_msb = addSPIRVInst(spv::OpExtInst, Ops);

    Constant *thirty_one = ConstantInt::get(
        Call->getType(), Call->getType()->getScalarSizeInBits() - 1);
    Ops.clear();
    Ops << Call->getType() << thirty_one << find_msb;
    return addSPIRVInst(spv::OpISub, Ops);
  }
  case Intrinsic::cttz: {
    // Implement as:
    // lsb = FindILsb x
    // res = lsb == -1 ? width : lsb
    //
    // Ignore the second operand of llvm.cttz.
    SPIRVOperandVec Ops;
    Ops << Call->getType() << getOpExtInstImportID()
        << glsl::ExtInst::ExtInstFindILsb << Call->getArgOperand(0);
    auto find_lsb = addSPIRVInst(spv::OpExtInst, Ops);

    auto neg_one = Constant::getAllOnesValue(Call->getType());
    auto i1_ty = Call->getType()->getWithNewBitWidth(1);
    auto width = ConstantInt::get(Call->getType(),
                                  Call->getType()->getScalarSizeInBits());

    Ops.clear();
    Ops << i1_ty << find_lsb << neg_one;
    auto cmp = addSPIRVInst(spv::OpIEqual, Ops);

    Ops.clear();
    Ops << Call->getType() << cmp << width << find_lsb;
    return addSPIRVInst(spv::OpSelect, Ops);
  }
  case Intrinsic::ctpop: {
    return GeneratePopcount(Call->getType(), Call->getArgOperand(0), Context);
  }
  case Intrinsic::bitreverse: {
    SPIRVOperandVec Ops;
    Ops << Call->getType() << Call->getArgOperand(0);
    return addSPIRVInst(spv::OpBitReverse, Ops);
  }
  case Intrinsic::fabs: {
    return GenerateFabs(Call->getOperand(0));
  }
  case Intrinsic::minimumnum: {
    SPIRVOperandVec Ops;
    Ops << Call->getType() << getOpExtInstImportID()
        << glsl::ExtInst::ExtInstFMin;
    if (ShouldFlushDenorms(Call->getType())) {
      Ops << GenerateCanonicalize(Call->getArgOperand(0))
          << GenerateCanonicalize(Call->getArgOperand(1));
    } else {
      Ops << Call->getArgOperand(0) << Call->getArgOperand(1);
    }
    return addSPIRVInst(spv::OpExtInst, Ops);
  }
  case Intrinsic::maximumnum: {
    SPIRVOperandVec Ops;
    Ops << Call->getType() << getOpExtInstImportID()
        << glsl::ExtInst::ExtInstFMax;
    if (ShouldFlushDenorms(Call->getType())) {
      Ops << GenerateCanonicalize(Call->getArgOperand(0))
          << GenerateCanonicalize(Call->getArgOperand(1));
    } else {
      Ops << Call->getArgOperand(0) << Call->getArgOperand(1);
    }
    return addSPIRVInst(spv::OpExtInst, Ops);
  }
  case Intrinsic::memcpy: {
    // This should only appear if untyped pointers are enabled.
    assert(Option::UntypedPointers());
    auto Dst = Call->getArgOperand(0);
    auto Src = Call->getArgOperand(1);
    auto Volatile =
        dyn_cast<ConstantInt>(Call->getArgOperand(3))->getZExtValue();

    uint32_t DstAlign = 0;
    uint32_t SrcAlign = 0;
    if (Call->paramHasAttr(0, Attribute::AttrKind::Alignment)) {
      DstAlign = static_cast<uint32_t>(
          Call->getParamAttr(0, Attribute::AttrKind::Alignment)
              .getValueAsInt());
    }
    if (Call->paramHasAttr(1, Attribute::AttrKind::Alignment)) {
      SrcAlign = static_cast<uint32_t>(
          Call->getParamAttr(1, Attribute::AttrKind::Alignment)
              .getValueAsInt());
    }

    SPIRVOperandVec Ops;
    Ops << Dst << Src << Call->getArgOperand(2);
    if (Option::SpvVersion() < Option::SPIRVVersion::SPIRV_1_4) {
      // Before 1.4, memory operands affected both Dst and Src so use the
      // smaller alignment.
      uint32_t align = std::min(DstAlign, SrcAlign);
      uint32_t memOp = 0;
      if (Volatile)
        memOp |= 0x1;
      if (align != 0)
        memOp |= 0x2;
      Ops << memOp << align;
    } else {
      uint32_t DstMemOp = 0;
      if (Volatile)
        DstMemOp |= 0x1;
      if (DstAlign != 0)
        DstMemOp |= 0x2;

      uint32_t SrcMemOp = 0;
      if (Volatile)
        SrcMemOp |= 0x1;
      if (SrcAlign != 0)
        SrcMemOp |= 0x2;

      Ops << DstMemOp << DstAlign << SrcMemOp << SrcAlign;
    }
    return addSPIRVInst(spv::OpCopyMemorySized, Ops);
  }
  case Intrinsic::canonicalize: {
    return GenerateCanonicalize(Call->getArgOperand(0));
  }

  default:
    break;
  }

  switch (func_type) {
  case Builtins::kFabs: {
    return GenerateFabs(Call->getOperand(0));
  }
  case Builtins::kPopcount: {
    return GeneratePopcount(Call->getType(), Call->getOperand(0), Context);
  }
  case Builtins::kShuffle: {
    auto Src = Call->getOperand(0);
    auto Mask = Call->getOperand(1);
    auto Ty = Call->getType();

    RID = GenerateShuffle2FromCall(Ty, Src, Src, Mask);
    break;
  }
  case Builtins::kShuffle2: {
    auto SrcA = Call->getOperand(0);
    auto SrcB = Call->getOperand(1);
    auto Mask = Call->getOperand(2);
    auto Ty = Call->getType();

    RID = GenerateShuffle2FromCall(Ty, SrcA, SrcB, Mask);
    break;
  }
  case Builtins::kNativeDivide: {
    SPIRVOperandVec Ops;
    Ops << Call->getType() << Call->getOperand(0) << Call->getOperand(1);
    RID = addSPIRVInst(spv::OpFDiv, Ops);
    break;
  }
  case Builtins::kDot:
  case Builtins::kIDotAccSat:
  case Builtins::kIDotAccSatPackedUUU:
  case Builtins::kIDotAccSatPackedSSS:
  case Builtins::kIDotAccSatPackedUSS:
  case Builtins::kIDotAccSatPackedSUS:
  case Builtins::kIDotPackedUUU:
  case Builtins::kIDotPackedSSS:
  case Builtins::kIDotPackedUSS:
  case Builtins::kIDotPackedSUS:
  case Builtins::kArmDotAcc: {
    RID = GenerateIntegerDot(Call, func_info);
    break;
  }
  default: {
    glsl::ExtInst EInst = Builtins::getDirectOrIndirectExtInstEnum(func_info);

    if (EInst && EInst == glsl::ExtInst::ExtInstFma &&
        clspv::Option::SupportsFmaKHR(Call->getType()->getScalarSizeInBits())) {
      addCapability(spv::CapabilityFMAKHR);
      SPIRVOperandVec Ops;
      Ops << Call->getType() << Call->getOperand(0) << Call->getOperand(1)
          << Call->getOperand(2);
      RID = addSPIRVInst(spv::OpFmaKHR, Ops);
    } else if (EInst && EInst == glsl::ExtInst::ExtInstFma &&
               !clspv::Option::UnsafeMath() && !clspv::Option::ClMadEnable() &&
               clspv::Option::UseNativeBuiltins().count(
                   clspv::Builtins::BuiltinType::kFma) == 0 &&
               Call->getType()->getScalarType()->isHalfTy()) {
      Type *FP16Ty = Call->getType();

      Type *FP32Ty = Type::getFloatTy(Context);
      if (FP16Ty->isVectorTy()) {
        FP32Ty = FixedVectorType::get(
            FP32Ty, dyn_cast<FixedVectorType>(FP16Ty)->getNumElements());
      }

      SPIRVOperandVec Ops;
      Ops.clear();
      Ops << FP32Ty << Call->getOperand(0);
      auto a_f32 = addSPIRVInst(spv::OpFConvert, Ops);

      Ops.clear();
      Ops << FP32Ty << Call->getOperand(1);
      auto b_f32 = addSPIRVInst(spv::OpFConvert, Ops);

      Ops.clear();
      Ops << FP32Ty << Call->getOperand(2);
      auto c_f32 = addSPIRVInst(spv::OpFConvert, Ops);

      Ops.clear();
      Ops << FP32Ty << getOpExtInstImportID() << EInst << a_f32 << b_f32
          << c_f32;
      auto fma_f32 = addSPIRVInst(spv::OpExtInst, Ops);

      Ops.clear();
      Ops << FP16Ty << fma_f32;
      RID = addSPIRVInst(spv::OpFConvert, Ops);

      // Do not replace functions with implementations.
    } else if (EInst && Call->getCalledFunction()->isDeclaration()) {
      SPIRVID ExtInstImportID = getOpExtInstImportID();

      //
      // Generate OpExtInst.
      //

      // Ops[0] = Result Type ID
      // Ops[1] = Set ID (OpExtInstImport ID)
      // Ops[2] = Instruction Number (Literal Number)
      // Ops[3] ... Ops[n] = Operand 1, ... , Operand n
      SPIRVOperandVec Ops;

      Ops << Call->getType() << ExtInstImportID << EInst;

      switch (EInst) {
      case glsl::ExtInst::ExtInstSAbs:
        // llvm.abs has a second argument that will not translate to the spirv
        // code.
        Ops << Call->getOperand(0);
        break;
      case glsl::ExtInst::ExtInstFMin:
      case glsl::ExtInst::ExtInstFMax:
      case glsl::ExtInst::ExtInstNMin:
      case glsl::ExtInst::ExtInstNMax: {
        if (ShouldFlushDenorms(Call->getType())) {
          for (auto &use : Call->args()) {
            Ops << GenerateCanonicalize(use.get());
          }
          break;
        }
        for (auto &use : Call->args()) {
          Ops << use.get();
        }
        break;
      }
      default:
        for (auto &use : Call->args()) {
          Ops << use.get();
        }
        break;
      }

      RID = addSPIRVInst(spv::OpExtInst, Ops);

      const auto IndirectExtInst = Builtins::getIndirectExtInstEnum(func_info);
      if (IndirectExtInst != kGlslExtInstBad) {
        // Generate one more instruction that uses the result of the extended
        // instruction.  Its result id is one more than the id of the
        // extended instruction.
        auto generate_extra_inst = [this, &Call, &RID](spv::Op opcode,
                                                       Constant *constant) {
          //
          // Generate instruction like:
          //   result = opcode constant <extinst-result>
          //
          // Ops[0] = Result Type ID
          // Ops[1] = Operand 0 ;; the constant, suitably splatted
          // Ops[2] = Operand 1 ;; the result of the extended instruction
          SPIRVOperandVec Ops;

          Type *resultTy = Call->getType();

          if (auto *vectorTy = dyn_cast<VectorType>(resultTy)) {
            constant =
                ConstantVector::getSplat(vectorTy->getElementCount(), constant);
          }
          Ops << resultTy << constant << RID;

          RID = addSPIRVInst(opcode, Ops);
        };

        switch (IndirectExtInst) {
        case glsl::ExtInstAcos:  // Implementing acospi
        case glsl::ExtInstAsin:  // Implementing asinpi
        case glsl::ExtInstAtan:  // Implementing atanpi
        case glsl::ExtInstAtan2: // Implementing atan2pi
          generate_extra_inst(
              spv::OpFMul,
              ConstantFP::get(Call->getType()->getScalarType(), kOneOverPi));
          break;

        default:
          assert(false && "internally inconsistent");
        }
      }
    } else {
      switch (Call->getIntrinsicID()) {
      // These LLVM intrinsics have no SPV equivalent.
      // Because they are optimiser hints, we can safely discard them.
      case Intrinsic::experimental_noalias_scope_decl:
        break;
      default:
        // A real function call (not builtin)
        // Call instruction is deferred because it needs function's ID.
        RID = addSPIRVPlaceholder(Call);
        break;
      }
    }

    break;
  }
  }

  return RID;
}

static bool isPowerOfTwo(Value *v) {
  auto isScalarPowerOfTwo = [](Value *val) -> bool {
    if (auto *CFP = dyn_cast<ConstantFP>(val)) {
      const auto &APF = CFP->getValueAPF();
      return APF.isNormal() && APF.getExactLog2Abs() != INT_MIN;
    }
    return false;
  };
  if (!v) {
    return false;
  }
  if (auto *VecTy = dyn_cast<FixedVectorType>(v->getType())) {
    auto *C = dyn_cast<Constant>(v);
    if (!C) {
      return false;
    }
    for (unsigned i = 0; i < VecTy->getNumElements(); ++i) {
      Constant *Elt = C->getAggregateElement(i);
      if (!Elt || !isScalarPowerOfTwo(Elt)) {
        return false;
      }
    }
    return true;
  }
  return isScalarPowerOfTwo(v);
}

bool SPIRVProducerPassImpl::ShouldHackFMulFlushToZero(Type *Ty) const {
  Type *ScalarTy = Ty->getScalarType();
  if (ScalarTy->isHalfTy()) {
    return clspv::Option::HackFMulFlushToZero(
        clspv::Option::FloatingPointType::fp16);
  } else if (ScalarTy->isFloatTy()) {
    return clspv::Option::HackFMulFlushToZero(
        clspv::Option::FloatingPointType::fp32);
  } else if (ScalarTy->isDoubleTy()) {
    return clspv::Option::HackFMulFlushToZero(
        clspv::Option::FloatingPointType::fp64);
  }
  return false;
}

void SPIRVProducerPassImpl::GenerateInstruction(Instruction &I) {
  ValueMapType &VMap = getValueMap();
  DIFileMap &DbgDIFileMap = getDebugDIFileMap();
  BBDILocMap &LastDILocInBB = getBBDILocMap();
  LLVMContext &Context = module->getContext();

  SPIRVID RID;

  auto md = I.getMetadata("dbg");
  if (md && isa<llvm::DILocation>(md)) {
    auto loc = dyn_cast<llvm::DILocation>(md);
    auto line = loc->getLine();
    auto column = loc->getColumn();
    auto file = loc->getFile();
    if (DbgDIFileMap.find(file) == DbgDIFileMap.end()) {
      DbgDIFileMap[file] = addSPIRVInst<kDebug>(
          spv::OpString, file->getFilename().str().c_str());
    }
    auto find = LastDILocInBB.find(I.getParent());
    if (!(find != LastDILocInBB.end() && find->second.first == line &&
          find->second.second == column)) {
      SPIRVOperandVec Ops;
      Ops << DbgDIFileMap[file] << line << column;
      addSPIRVInst(spv::OpLine, Ops);
      LastDILocInBB[I.getParent()] = std::make_pair(line, column);
    }
  } else if (clspv::Option::DebugInfo()) {
    LastDILocInBB.erase(I.getParent());
    addSPIRVInst(spv::OpNoLine);
  }

  if (clspv::Option::DecorateNonUniform() &&
      I.getOpcode() != Instruction::PHI) {
    for (auto &op : I.operands()) {
      if (!isPointerUniform(op)) {
        setNonUniformPointers();

        SPIRVOperandVec Ops;
        auto id = getSPIRVValue(op);
        if (DecoratedNonUniform.insert(id.get()).second) {
          Ops << id << spv::DecorationNonUniform;
          addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
        }
      }
    }
  }

  switch (I.getOpcode()) {
  default: {
    if (Instruction::isCast(I.getOpcode())) {
      //
      // Generate SPIRV instructions for cast operators.
      //

      auto Ty = I.getType();
      auto OpTy = I.getOperand(0)->getType();
      auto toI8 = Ty == Type::getInt8Ty(Context);
      auto fromI32 = OpTy == Type::getInt32Ty(Context);
      // Handle zext, sext, uitofp, and sitofp with i1 type specially.
      if ((I.getOpcode() == Instruction::ZExt ||
           I.getOpcode() == Instruction::SExt ||
           I.getOpcode() == Instruction::UIToFP ||
           I.getOpcode() == Instruction::SIToFP) &&
          OpTy->isIntOrIntVectorTy(1)) {
        //
        // Generate OpSelect.
        //

        // Ops[0] = Result Type ID
        // Ops[1] = Condition ID
        // Ops[2] = True Constant ID
        // Ops[3] = False Constant ID
        SPIRVOperandVec Ops;

        Ops << I.getType() << I.getOperand(0);

        if (I.getOpcode() == Instruction::ZExt) {
          Ops << ConstantInt::get(I.getType(), 1);
        } else if (I.getOpcode() == Instruction::SExt) {
          Ops << ConstantInt::getSigned(I.getType(), -1);
        } else if (I.getOpcode() == Instruction::UIToFP) {
          Ops << ConstantFP::get(I.getType(), 1.0);
        } else if (I.getOpcode() == Instruction::SIToFP) {
          Ops << ConstantFP::get(I.getType(), -1.0);
        }

        if (I.getOpcode() == Instruction::ZExt) {
          Ops << Constant::getNullValue(I.getType());
        } else if (I.getOpcode() == Instruction::SExt) {
          Ops << Constant::getNullValue(I.getType());
        } else {
          Ops << ConstantFP::get(I.getType(), 0.0);
        }

        RID = addSPIRVInst(spv::OpSelect, Ops);
      } else if (!clspv::Option::Int8Support() && Ty->isFPOrFPVectorTy() &&
                 OpTy->isIntOrIntVectorTy(8) &&
                 I.getOpcode() == Instruction::SIToFP) {
        SPIRVOperandVec Ops;
        Ops << OpTy << I.getOperand(0)
            << getSPIRVConstant(ConstantInt::get(OpTy, 24));
        RID = addSPIRVInst(spv::OpShiftLeftLogical, Ops);
        Ops.clear();
        Ops << OpTy << RID << getSPIRVConstant(ConstantInt::get(OpTy, 24));
        RID = addSPIRVInst(spv::OpShiftRightArithmetic, Ops);
        Ops.clear();
        Ops << Ty << RID;
        RID = addSPIRVInst(spv::OpConvertSToF, Ops);
      } else if (!clspv::Option::Int8Support() && Ty->isIntOrIntVectorTy(32) &&
                 OpTy->isIntOrIntVectorTy(8) &&
                 I.getOpcode() == Instruction::SExt) {
        SPIRVOperandVec Ops;
        Ops << Ty << I.getOperand(0)
            << getSPIRVConstant(ConstantInt::get(Ty, 24));
        RID = addSPIRVInst(spv::OpShiftLeftLogical, Ops);
        Ops.clear();
        Ops << Ty << RID << getSPIRVConstant(ConstantInt::get(Ty, 24));
        RID = addSPIRVInst(spv::OpShiftRightArithmetic, Ops);
      } else if (!clspv::Option::Int8Support() &&
                 I.getOpcode() == Instruction::Trunc && fromI32 && toI8) {
        // The SPIR-V target type is a 32-bit int.  Keep only the bottom
        // 8 bits.
        // Before:
        //   %result = trunc i32 %a to i8
        // After
        //   %result = OpBitwiseAnd %uint %a %uint_255

        SPIRVOperandVec Ops;

        Ops << OpTy << I.getOperand(0) << getSPIRVInt32Constant(255);

        RID = addSPIRVInst(spv::OpBitwiseAnd, Ops);
      } else if (!clspv::Option::Int8Support() &&
                 I.getOpcode() == Instruction::BitCast &&
                 ((Ty->isIntOrIntVectorTy(8) && OpTy->isIntOrIntVectorTy(32)) ||
                  (Ty->isIntOrIntVectorTy(32) &&
                   OpTy->isIntOrIntVectorTy(8)))) {
        // Without int8 support, <4 x i8> is represented as i32.
        // The only valid bitcast involving i8 and i32 is <4 x i8> <--> i32
        // because Vulkan's max vector size is 4 and the total bitwidth must
        // match.
        RID = getSPIRVValue(I.getOperand(0));
      } else if (Ty->isIntOrIntVectorTy(1) &&
                 I.getOpcode() == Instruction::Trunc) {
        // We usually map trunc to OpUConvert.
        // But OpUConvert only takes integer types in input, not boolean type.
        // Instead use OpINotEqual.
        SPIRVOperandVec Ops;
        Ops << OpTy << I.getOperand(0)
            << getSPIRVValue(ConstantInt::get(OpTy, 1));
        auto oAnd1 = addSPIRVInst(spv::OpBitwiseAnd, Ops);

        Ops.clear();
        Ops << Ty << oAnd1 << getSPIRVValue(ConstantInt::get(OpTy, 0));
        RID = addSPIRVInst(spv::OpINotEqual, Ops);
      } else {
        // Ops[0] = Result Type ID
        // Ops[1] = Source Value ID
        SPIRVOperandVec Ops;

        Type *Ty;
        if (I.getOpcode() == Instruction::IntToPtr ||
            I.getOpcode() == Instruction::AddrSpaceCast) {
          Ty = clspv::InferType(&I, module->getContext(), &InferredTypeCache);
          assert(Ty);
          Ops << getSPIRVPointerType(I.getType(), Ty);
        } else {
          Ty = I.getType();
          Ops << Ty;
        }
        Ops << getSPIRVValue(I.getOperand(0), Ty);

        auto Op = GetSPIRVCastOpcode(I);
        RID = addSPIRVInst(Op, Ops);

        if (Op == spv::OpConvertSToF || Op == spv::OpConvertUToF ||
            Op == spv::OpFConvert) {
          // signal that we might want to the set the execution mode
          setConvertToF();
        }

        if (clspv::Option::HackConvertToFloat() &&
            (Op == spv::OpConvertSToF || Op == spv::OpConvertUToF ||
             Op == spv::OpFConvert)) {
          auto copysign_zero = GenerateCopySignZero(RID, I.getType());
          Ops.clear();
          Ops << I.getType() << RID << copysign_zero;
          RID = addSPIRVInst(spv::OpFAdd, Ops);
        }
      }
    } else if (isa<BinaryOperator>(I)) {
      //
      // Generate SPIRV instructions for binary operators.
      //

      // Handle xor with i1 type specially.
      if (I.getOpcode() == Instruction::Xor &&
          I.getType() == Type::getInt1Ty(Context) &&
          ((isa<ConstantInt>(I.getOperand(0)) &&
            !cast<ConstantInt>(I.getOperand(0))->isZero()) ||
           (isa<ConstantInt>(I.getOperand(1)) &&
            !cast<ConstantInt>(I.getOperand(1))->isZero()))) {
        //
        // Generate OpLogicalNot.
        //
        // Ops[0] = Result Type ID
        // Ops[1] = Operand
        SPIRVOperandVec Ops;

        Ops << I.getType();

        Value *CondV = I.getOperand(0);
        if (isa<Constant>(I.getOperand(0))) {
          CondV = I.getOperand(1);
        }
        Ops << CondV;

        RID = addSPIRVInst(spv::OpLogicalNot, Ops);
      } else if (I.getOpcode() == Instruction::SRem) {
        // We cannot map SRem to OpSRem because:
        // For the OpSRem and OpSMod instructions, if either operand is negative
        // the result is undefined.
        // https://www.khronos.org/registry/vulkan/specs/1.3-extensions/html/chap46.html#spirvenv-precision-operation
        //
        // Let's implement signed modulus like that:
        // SRem(x, y) {
        //   rem = (abs(x) % abs(y))
        //   return (x > 0 ? rem : ((rem ^ -1) + 1)); // (x > 0 ? rem : rem *
        //   -1);
        // }

        SPIRVOperandVec Ops;
        Type *Ty = I.getType();
        Type *boolTy = Ty->getInt1Ty(Ty->getContext());
        if (Ty->isVectorTy()) {
          boolTy = VectorType::get(boolTy, dyn_cast<VectorType>(Ty));
        }

        auto Cst0 = ConstantInt::get(Ty, 0, true);
        auto Cst1 = ConstantInt::get(Ty, 1, true);
        auto Cstm1 = ConstantInt::get(Ty, -1, true);
        auto X = I.getOperand(0);
        auto Y = I.getOperand(1);

        // abs(x)
        Ops << Ty << getOpExtInstImportID() << glsl::ExtInst::ExtInstSAbs << X;
        auto AbsX = addSPIRVInst(spv::OpExtInst, Ops);

        // abs(y)
        Ops.clear();
        Ops << Ty << getOpExtInstImportID() << glsl::ExtInst::ExtInstSAbs << Y;
        auto AbsY = addSPIRVInst(spv::OpExtInst, Ops);

        // abs(x) % abs(y)
        Ops.clear();
        Ops << Ty << AbsX << AbsY;
        auto Mod = addSPIRVInst(spv::OpUMod, Ops);

        // x > 0
        Ops.clear();
        Ops << boolTy << X << Cst0;
        auto Cmp = addSPIRVInst(spv::OpSGreaterThan, Ops);

        Ops.clear();
        Ops << Ty << Mod << Cstm1;
        auto Xor = addSPIRVInst(spv::OpBitwiseXor, Ops);

        Ops.clear();
        Ops << Ty << Xor << Cst1;
        auto Modm1 = addSPIRVInst(spv::OpIAdd, Ops);

        Ops.clear();
        Ops << Ty << Cmp << Mod << Modm1;
        RID = addSPIRVInst(spv::OpSelect, Ops);
      } else if (I.getOpcode() == Instruction::FDiv && !Option::UnsafeMath() &&
                 I.getType()->getScalarType()->isFloatTy()) {
        // float div(float a, float b) {
        //   if (abs(b) > 0x1.0p+126f) {
        //     c = 0x1.0p-4f;
        //   } else if (abs(b) < 0x1.0p-126f) {
        //     c = 0x1.0p+24f;
        //   } else {
        //     c = 1.0;
        //   }
        //   return (a / (b * c)) * c;
        // }
        Type *FPTy = I.getType();
        auto a = getSPIRVValue(I.getOperand(0));
        auto b = getSPIRVValue(I.getOperand(1));
        bool is_vector = FPTy->isVectorTy();
        SPIRVOperandVec Ops;
        SPIRVID Ty = getSPIRVType(FPTy);

        const float fmax_val = 0x1.0p+126f;
        const float fmin_val = 0x1.0p-126f;
        const float c_max_val = 0x1.0p-4f;
        const float c_min_val = 0x1.0p+24f;
        SPIRVID divisor, c;

        // Try to optimize it if b is constant.
        if (auto b_cst = dyn_cast<Constant>(I.getOperand(1))) {
          SmallVector<Constant *, 2> csts;
          if (is_vector) {
            for (unsigned i = 0;
                 i < cast<FixedVectorType>(FPTy)->getNumElements(); i++) {
              csts.push_back(b_cst->getAggregateElement(i));
            }
          } else {
            csts.push_back(b_cst);
          }

          SmallVector<Constant *, 2> divisor_vec;
          SmallVector<Constant *, 2> c_vec;
          Type *ScalarTy = FPTy->getScalarType();
          bool c_vec_needed = false;
          bool vector_cst = true;
          for (auto b_elem : csts) {
            float b_elem_val = 1.0f;
            if (auto b_elem_cst_fp = dyn_cast<ConstantFP>(b_elem)) {
              b_elem_val = b_elem_cst_fp->getValue().convertToFloat();
            } else if (!isa<PoisonValue>(b_elem)) {
              vector_cst = false;
              break;
            }
            if (b_elem_val > fmax_val) {
              divisor_vec.push_back(
                  ConstantFP::get(ScalarTy, b_elem_val * c_max_val));
              c_vec.push_back(ConstantFP::get(ScalarTy, c_max_val));
              c_vec_needed = true;
            } else if (b_elem_val < fmin_val) {
              divisor_vec.push_back(
                  ConstantFP::get(ScalarTy, b_elem_val * c_min_val));
              c_vec.push_back(ConstantFP::get(ScalarTy, c_min_val));
              c_vec_needed = true;
            } else {
              divisor_vec.push_back(ConstantFP::get(ScalarTy, b_elem_val));
              c_vec.push_back(ConstantFP::get(ScalarTy, 1.0));
            }
          }
          if (vector_cst && c_vec_needed) {
            if (is_vector) {
              divisor = getSPIRVConstant(ConstantVector::get(divisor_vec));
              c = getSPIRVConstant(ConstantVector::get(c_vec));
            } else {
              divisor = getSPIRVConstant(divisor_vec[0]);
              c = getSPIRVConstant(c_vec[0]);
            }
          } else if (vector_cst) {
            divisor = getSPIRVConstant(b_cst);
          }
        }

        if (!divisor.isValid()) {
          Type *boolTy = IntegerType::get(module->getContext(), 1);
          if (is_vector) {
            boolTy = VectorType::get(boolTy, dyn_cast<VectorType>(FPTy));
          }
          SPIRVID c_max = getSPIRVConstant(ConstantFP::get(FPTy, c_max_val));
          SPIRVID c_min = getSPIRVConstant(ConstantFP::get(FPTy, c_min_val));
          SPIRVID max = getSPIRVConstant(ConstantFP::get(FPTy, fmax_val));
          SPIRVID min = getSPIRVConstant(ConstantFP::get(FPTy, fmin_val));
          SPIRVID c_one = getSPIRVConstant(ConstantFP::get(FPTy, 1.0));

          SPIRVID abs_b = GenerateFabs(I.getOperand(1));

          Ops.clear();
          Ops << getSPIRVType(boolTy) << abs_b << max;
          SPIRVID cond_max = addSPIRVInst(spv::OpFOrdGreaterThan, Ops);
          Ops.clear();
          Ops << getSPIRVType(boolTy) << abs_b << min;
          SPIRVID cond_min = addSPIRVInst(spv::OpFOrdLessThan, Ops);

          Ops.clear();
          Ops << Ty << cond_max << c_max << c_one;
          c = addSPIRVInst(spv::OpSelect, Ops);
          Ops.clear();
          Ops << Ty << cond_min << c_min << c;
          c = addSPIRVInst(spv::OpSelect, Ops);

          Ops.clear();
          Ops << Ty << b << c;
          divisor = addSPIRVInst(spv::OpFMul, Ops);
        }

        Ops.clear();
        Ops << Ty << a << divisor;
        RID = addSPIRVInst(spv::OpFDiv, Ops);

        if (c.isValid()) {
          Ops.clear();
          Ops << Ty << RID << c;
          RID = addSPIRVInst(spv::OpFMul, Ops);
        }
      } else if (I.getOpcode() == Instruction::FDiv &&
                 I.getType()->getScalarType()->isHalfTy() &&
                 !clspv::Option::UnsafeMath()) {
        Type *FP16Ty = I.getType();
        Type *FP32Ty = Type::getFloatTy(Context);
        if (FP16Ty->isVectorTy()) {
          FP32Ty = FixedVectorType::get(
              FP32Ty, dyn_cast<FixedVectorType>(FP16Ty)->getNumElements());
        }

        SPIRVOperandVec Ops;
        Ops.clear();
        Ops << FP32Ty << I.getOperand(0);
        auto a_f32 = addSPIRVInst(spv::OpFConvert, Ops);

        Ops.clear();
        Ops << FP32Ty << I.getOperand(1);
        auto b_f32 = addSPIRVInst(spv::OpFConvert, Ops);

        Ops.clear();
        Ops << FP32Ty << a_f32 << b_f32;
        auto div_f32 = addSPIRVInst(spv::OpFDiv, Ops);

        Ops.clear();
        Ops << FP16Ty << div_f32;
        RID = addSPIRVInst(spv::OpFConvert, Ops);
      } else {
        // Ops[0] = Result Type ID
        // Ops[1] = Operand 0
        // Ops[2] = Operand 1
        SPIRVOperandVec Ops;

        auto opcode = GetSPIRVBinaryOpcode(I);
        auto a = getSPIRVValue(I.getOperand(0));
        auto b = getSPIRVValue(I.getOperand(1));
        bool shouldCanonicalize =
            opcode == spv::OpFMul && ShouldHackFMulFlushToZero(I.getType()) &&
            !isPowerOfTwo(I.getOperand(0)) && !isPowerOfTwo(I.getOperand(1));
        if (shouldCanonicalize) {
          a = GenerateCanonicalize(a, I.getType());
          b = GenerateCanonicalize(b, I.getType());
        }

        Ops << I.getType() << a << b;
        RID = addSPIRVInst(opcode, Ops);
        if (shouldCanonicalize) {
          RID = GenerateCanonicalize(RID, I.getType());
        }
      }
    } else if (I.getOpcode() == Instruction::FNeg) {
      // The only unary operator.
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Operand 0
      SPIRVOperandVec Ops;

      Ops << I.getType() << I.getOperand(0);
      RID = addSPIRVInst(spv::OpFNegate, Ops);
    } else if (I.getOpcode() == Instruction::Unreachable) {
      RID = addSPIRVInst(spv::OpUnreachable);
    } else {
      I.print(errs());
      llvm_unreachable("Unsupported instruction???");
    }
    break;
  }
  case Instruction::GetElementPtr: {
    auto &GlobalConstArgSet = getGlobalConstArgSet();

    //
    // Generate OpAccessChain.
    //
    GetElementPtrInst *GEP = cast<GetElementPtrInst>(&I);

    //
    // Generate OpAccessChain.
    //

    // Ops[0] = Result Type ID
    // Ops[1] = Base ID
    // Ops[2] ... Ops[n] = Indexes ID
    SPIRVOperandVec Ops;

    PointerType *ResultType = cast<PointerType>(GEP->getType());
    if (GEP->getPointerAddressSpace() == AddressSpace::ModuleScopePrivate ||
        GlobalConstArgSet.count(GEP->getPointerOperand())) {
      // Use pointer type with private address space for global constant.
      ResultType = PointerType::get(module->getContext(),
                                    AddressSpace::ModuleScopePrivate);
      GlobalConstArgSet.insert(GEP);
    }

    const bool untyped =
        Option::UntypedPointerAddressSpace(ResultType->getAddressSpace());

    // TODO(dneto): Simplify the following?

    //
    // Follows below rules for gep.
    //
    // 1. If gep's first index is 0 generate OpAccessChain and ignore gep's
    //    first index.
    // 2. If gep's first index is not 0, generate OpPtrAccessChain and use gep's
    //    first index.
    // 3. If gep's first index is not constant, generate OpPtrAccessChain and
    //    use gep's first index.
    // 4. If it is not above case 1, 2 and 3, generate OpAccessChain and use
    //    gep's first index.
    //
    spv::Op Opcode = spv::OpAccessChain;
    unsigned offset = 0;
    if (ConstantInt *CstInt = dyn_cast<ConstantInt>(GEP->getOperand(1))) {
      if (CstInt->getZExtValue() == 0) {
        offset = 1;
      } else if (CstInt->getZExtValue() != 0) {
        Opcode = spv::OpPtrAccessChain;
      }
    } else {
      Opcode = spv::OpPtrAccessChain;
    }

    bool needs_array = false;
    if (Opcode == spv::OpPtrAccessChain) {
      // Shader validation in the SPIR-V spec requires that the base pointer to
      // OpPtrAccessChain (in StorageBuffer storage class) be decorated with
      // ArrayStride.
      auto address_space = ResultType->getAddressSpace();
      switch (GetStorageClass(address_space)) {
      case spv::StorageClassStorageBuffer:
      case spv::StorageClassPhysicalStorageBuffer:
        if (!untyped) {
          // Save the type to generate an ArrayStride decoration later, but
          // assume opaque pointers may be present so also cache the SPIRVID for
          // the type and the stride value.
          getTypesNeedingArrayStride().insert(StrideType(
              static_cast<uint32_t>(GetTypeAllocSize(
                  GEP->getSourceElementType(), module->getDataLayout())),
              getSPIRVPointerType(GEP->getPointerOperand()->getType(),
                                  GEP->getSourceElementType())));
        } else {
          // In order to avoid tracking multiple untyped pointers in the same
          // storage class with different ArrayStride decorations, generate a
          // normal access chain based on an arrayed source element type.
          needs_array = true;
        }
        break;
      case spv::StorageClassWorkgroup:
        if (untyped) {
          needs_array = true;
        }
        break;
      case spv::StorageClassUniform:
      case spv::StorageClassPushConstant:
        if (untyped) {
          needs_array = true;
        } else {
          llvm_unreachable(
              "OpPtrAccessChain is not supported for this storage class");
        }
        break;
      default:
        llvm_unreachable(
            "OpPtrAccessChain is not supported for this storage class");
        break;
      }
    }

    Ops << getSPIRVPointerType(ResultType, GEP->getResultElementType());
    if (untyped) {
      Opcode = spv::OpUntypedAccessChainKHR;
      SPIRVID base_type_id = getSPIRVType(GEP->getSourceElementType(), true);
      if (needs_array) {
        auto arrayTy = ArrayType::get(GEP->getSourceElementType(), 0);
        base_type_id = getSPIRVType(arrayTy, true);
      }
      Ops << base_type_id;
    } else if (Opcode == spv::OpPtrAccessChain) {
      auto address_space = ResultType->getAddressSpace();
      setVariablePointersCapabilities(address_space);
    }

    // Generate the base pointer.
    if (untyped) {
      Ops << GEP->getPointerOperand();
    } else {
      Ops << getSPIRVPointerOperand(GEP->getPointerOperand(),
                                    GEP->getSourceElementType());
    }

    for (auto II = GEP->idx_begin() + offset; II != GEP->idx_end(); II++) {
      Ops << *II;
    }

    RID = addSPIRVInst(Opcode, Ops);
    break;
  }
  case Instruction::ExtractValue: {
    ExtractValueInst *EVI = cast<ExtractValueInst>(&I);
    // Ops[0] = Result Type ID
    // Ops[1] = Composite ID
    // Ops[2] ... Ops[n] = Indexes (Literal Number)
    SPIRVOperandVec Ops;

    Ops << I.getType();

    Ops << EVI->getAggregateOperand();

    for (auto &Index : EVI->indices()) {
      Ops << Index;
    }

    RID = addSPIRVInst(spv::OpCompositeExtract, Ops);
    break;
  }
  case Instruction::InsertValue: {
    InsertValueInst *IVI = cast<InsertValueInst>(&I);
    // Ops[0] = Result Type ID
    // Ops[1] = Object ID
    // Ops[2] = Composite ID
    // Ops[3] ... Ops[n] = Indexes (Literal Number)
    SPIRVOperandVec Ops;

    Ops << I.getType() << IVI->getInsertedValueOperand()
        << IVI->getAggregateOperand();

    for (auto &Index : IVI->indices()) {
      Ops << Index;
    }

    RID = addSPIRVInst(spv::OpCompositeInsert, Ops);
    break;
  }
  case Instruction::Select: {
    //
    // Generate OpSelect.
    //

    // Ops[0] = Result Type ID
    // Ops[1] = Condition ID
    // Ops[2] = True Constant ID
    // Ops[3] = False Constant ID
    SPIRVOperandVec Ops;

    // Find SPIRV instruction for parameter type.
    auto Ty = I.getType();
    auto &GlobalConstArgSet = getGlobalConstArgSet();
    if (Ty->isPointerTy()) {
      if (GlobalConstArgSet.count(I.getOperand(1)) ||
          GlobalConstArgSet.count(I.getOperand(2))) {
        Ty = PointerType::get(module->getContext(),
                              AddressSpace::ModuleScopePrivate);
        GlobalConstArgSet.insert(&I);
      }
      // Selecting between pointers requires variable pointers.
      setVariablePointersCapabilities(Ty->getPointerAddressSpace());
      if (GetStorageClass(Ty->getPointerAddressSpace()) !=
              spv::StorageClassPhysicalStorageBuffer &&
          !hasVariablePointers() && !selectFromSameObject(&I)) {
        setVariablePointers();
      }
    }

    SPIRVID type_id;
    Type *DataTy = Ty;
    if (Ty->isPointerTy()) {
      DataTy = InferType(&I, module->getContext(), &InferredTypeCache);
      type_id = getSPIRVPointerType(Ty, DataTy);
    } else {
      type_id = getSPIRVType(Ty);
    }

    Ops << type_id << getSPIRVValue(I.getOperand(0), DataTy)
        << getSPIRVValue(I.getOperand(1), DataTy)
        << getSPIRVValue(I.getOperand(2), DataTy);

    RID = addSPIRVInst(spv::OpSelect, Ops);
    break;
  }
  case Instruction::ExtractElement: {
    // Handle <4 x i8> type manually.
    Type *CompositeTy = I.getOperand(0)->getType();
    if (is4xi8vec(CompositeTy)) {
      //
      // Generate OpShiftRightLogical and OpBitwiseAnd for extractelement with
      // <4 x i8>.
      //

      //
      // Generate OpShiftRightLogical
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Operand 0
      // Ops[2] = Operand 1
      //
      SPIRVOperandVec Ops;

      Ops << CompositeTy << I.getOperand(0);

      SPIRVID Op1ID = 0;
      if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(1))) {
        // Handle constant index.
        uint32_t Idx = static_cast<uint32_t>(CI->getZExtValue());
        Op1ID = getSPIRVInt32Constant(Idx * 8);
      } else {
        // Handle variable index.
        SPIRVOperandVec TmpOps;

        TmpOps << Type::getInt32Ty(Context) << I.getOperand(1)
               << getSPIRVInt32Constant(8);

        Op1ID = addSPIRVInst(spv::OpIMul, TmpOps);
      }
      Ops << Op1ID;

      SPIRVID ShiftID = addSPIRVInst(spv::OpShiftRightLogical, Ops);

      //
      // Generate OpBitwiseAnd
      //
      // Ops[0] = Result Type ID
      // Ops[1] = Operand 0
      // Ops[2] = Operand 1
      //
      Ops.clear();

      Ops << CompositeTy << ShiftID << getSPIRVInt32Constant(0xFF);

      RID = addSPIRVInst(spv::OpBitwiseAnd, Ops);
      break;
    }

    // Ops[0] = Result Type ID
    // Ops[1] = Composite ID
    // Ops[2] ... Ops[n] = Indexes (Literal Number)
    SPIRVOperandVec Ops;

    Ops << I.getType() << I.getOperand(0);

    spv::Op Opcode = spv::OpCompositeExtract;
    if (const ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(1))) {
      Ops << static_cast<uint32_t>(CI->getZExtValue());
    } else {
      Ops << I.getOperand(1);
      Opcode = spv::OpVectorExtractDynamic;
    }

    RID = addSPIRVInst(Opcode, Ops);
    break;
  }
  case Instruction::InsertElement: {
    // Handle <4 x i8> type manually.
    Type *CompositeTy = I.getOperand(0)->getType();
    if (is4xi8vec(CompositeTy)) {
      SPIRVID CstFFID = getSPIRVInt32Constant(0xFF);

      SPIRVID ShiftAmountID = 0;
      if (ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(2))) {
        // Handle constant index.
        uint32_t Idx = static_cast<uint32_t>(CI->getZExtValue());
        ShiftAmountID = getSPIRVInt32Constant(Idx * 8);
      } else {
        // Handle variable index.
        SPIRVOperandVec TmpOps;

        TmpOps << Type::getInt32Ty(Context) << I.getOperand(2)
               << getSPIRVInt32Constant(8);

        ShiftAmountID = addSPIRVInst(spv::OpIMul, TmpOps);
      }

      //
      // Generate mask operations.
      //

      // ShiftLeft mask according to index of insertelement.
      SPIRVOperandVec Ops;

      Ops << CompositeTy << CstFFID << ShiftAmountID;

      SPIRVID MaskID = addSPIRVInst(spv::OpShiftLeftLogical, Ops);

      // Inverse mask.
      Ops.clear();
      Ops << CompositeTy << MaskID;

      SPIRVID InvMaskID = addSPIRVInst(spv::OpNot, Ops);

      // Apply mask.
      Ops.clear();
      Ops << CompositeTy << I.getOperand(0) << InvMaskID;

      SPIRVID OrgValID = addSPIRVInst(spv::OpBitwiseAnd, Ops);

      // Create correct value according to index of insertelement.
      Ops.clear();
      Ops << CompositeTy << I.getOperand(1) << ShiftAmountID;

      SPIRVID InsertValID = addSPIRVInst(spv::OpShiftLeftLogical, Ops);

      // Insert value to original value.
      Ops.clear();
      Ops << CompositeTy << OrgValID << InsertValID;

      RID = addSPIRVInst(spv::OpBitwiseOr, Ops);
      break;
    }

    SPIRVOperandVec Ops;

    // Ops[0] = Result Type ID
    Ops << I.getType();

    spv::Op Opcode = spv::OpCompositeInsert;
    if (const ConstantInt *CI = dyn_cast<ConstantInt>(I.getOperand(2))) {
      const auto value = CI->getZExtValue();
      assert(value <= UINT32_MAX);
      // Ops[1] = Object ID
      // Ops[2] = Composite ID
      // Ops[3] ... Ops[n] = Indexes (Literal Number)
      Ops << I.getOperand(1) << I.getOperand(0) << static_cast<uint32_t>(value);
    } else {
      // Ops[1] = Composite ID
      // Ops[2] = Object ID
      // Ops[3] ... Ops[n] = Indexes (Literal Number)
      Ops << I.getOperand(0) << I.getOperand(1) << I.getOperand(2);
      Opcode = spv::OpVectorInsertDynamic;
    }

    RID = addSPIRVInst(Opcode, Ops);
    break;
  }
  case Instruction::ShuffleVector: {
    // Ops[0] = Result Type ID
    // Ops[1] = Vector 1 ID
    // Ops[2] = Vector 2 ID
    // Ops[3] ... Ops[n] = Components (Literal Number)
    SPIRVOperandVec Ops;

    if (!clspv::Option::Int8Support() && I.getType()->isIntOrIntVectorTy(8)) {
      // <4 x i8> is translated as i32. So shufflevector on <4 x i8> must be
      // implemented as bit operations.
      // This could be optimized, but it is a legacy support case.
      auto int32_ty = Type::getInt32Ty(module->getContext());
      auto tmp = getSPIRVConstant(Constant::getNullValue(int32_ty));
      const auto shuffle = dyn_cast<ShuffleVectorInst>(&I);
      uint32_t i = 0;
      for (auto mask : shuffle->getShuffleMask()) {
        auto op0 = I.getOperand(0);
        auto op1 = I.getOperand(1);
        auto vec_ty = dyn_cast<VectorType>(op0->getType());
        auto num_eles = vec_ty->getElementCount().getFixedValue();
        bool use_op0 = mask < (int)num_eles;
        auto op = use_op0 ? op0 : op1;
        auto mask_byte = mask % num_eles;
        if (!isa<UndefValue>(op) && !isa<PoisonValue>(op)) {
          Ops.clear();

          // This element goes in the |i|'th byte using the |mask_byte| byte of
          // |op|.
          const uint32_t bitmask = 0xff << (mask_byte * 8);
          auto bitmask_const =
              getSPIRVConstant(ConstantInt::get(int32_ty, bitmask));
          Ops << op->getType() << op << bitmask_const;
          auto and_mask = addSPIRVInst(spv::OpBitwiseAnd, Ops);
          Ops.clear();

          const int32_t shift_amount = i * 8 - mask_byte * 8;
          const bool shl = shift_amount > 0;
          if (shift_amount != 0) {
            Ops << op->getType() << and_mask
                << getSPIRVConstant(ConstantInt::get(int32_ty, shift_amount));
            and_mask = addSPIRVInst(
                shl ? spv::OpShiftLeftLogical : spv::OpShiftRightLogical, Ops);
            Ops.clear();
          }

          Ops << I.getType() << tmp << and_mask;
          tmp = addSPIRVInst(spv::OpBitwiseOr, Ops);
        }
        ++i;
      }
      RID = tmp;
    } else {
      Ops << I.getType() << I.getOperand(0) << I.getOperand(1);

      auto shuffle = cast<ShuffleVectorInst>(&I);
      SmallVector<int, 4> mask;
      shuffle->getShuffleMask(mask);
      for (auto i : mask) {
        if (i == PoisonMaskElem) {
          if (clspv::Option::HackUndef())
            // Use 0 instead of undef.
            Ops << 0;
          else
            // Undef for shuffle in SPIR-V.
            Ops << 0xffffffff;
        } else {
          Ops << i;
        }
      }

      RID = addSPIRVInst(spv::OpVectorShuffle, Ops);
    }
    break;
  }
  case Instruction::ICmp:
  case Instruction::FCmp: {
    CmpInst *CmpI = cast<CmpInst>(&I);

    // Pointer comparisons are only supported from SPIR-V 1.4 onwards.
    Type *ArgTy = CmpI->getOperand(0)->getType();
    if (isa<PointerType>(ArgTy)) {
      if (SpvVersion() >= SPIRVVersion::SPIRV_1_4) {
        SPIRVOperandVec Ops;
        auto *lhs = CmpI->getOperand(0);
        auto *rhs = CmpI->getOperand(1);
        auto *lhs_ty = clspv::InferType(lhs, Context, &InferredTypeCache);
        auto *rhs_ty = clspv::InferType(rhs, Context, &InferredTypeCache);
        SPIRVID cmp_lhs, cmp_rhs;
        // Need a better way to handle pointer constants
        if (!lhs_ty && !rhs_ty) {
          llvm_unreachable("neither pointer type can be inferred");
        }
        if (lhs_ty && isa<ConstantPointerNull>(rhs)) {
          cmp_lhs = getSPIRVValue(lhs);
          auto type_id = getSPIRVPointerType(lhs->getType(), lhs_ty);
          Ops.clear();
          Ops << type_id;
          cmp_rhs = addSPIRVInst<kConstants>(spv::Op::OpConstantNull, Ops);
        } else if (rhs_ty && isa<ConstantPointerNull>(rhs)) {
          cmp_rhs = getSPIRVValue(rhs);
          auto type_id = getSPIRVPointerType(rhs->getType(), rhs_ty);
          Ops.clear();
          Ops << type_id;
          cmp_lhs = addSPIRVInst<kConstants>(spv::Op::OpConstantNull, Ops);
        } else {
          cmp_lhs = getSPIRVValue(lhs);
          cmp_rhs = getSPIRVValue(rhs);
        }
        spv::Op Opcode;
        switch (CmpI->getPredicate()) {
        case CmpInst::ICMP_NE:
          Opcode = spv::OpPtrNotEqual;
          break;
        case CmpInst::ICMP_EQ:
          Opcode = spv::OpPtrEqual;
          break;
        case CmpInst::ICMP_UGT:
        case CmpInst::ICMP_UGE:
        case CmpInst::ICMP_ULT:
        case CmpInst::ICMP_ULE:
          Opcode = GetSPIRVPointerCmpOpcode(CmpI);
          Ops.clear();
          Ops << getSPIRVType(Type::getInt32Ty(Context)) << cmp_lhs << cmp_rhs;
          RID = addSPIRVInst(spv::OpPtrDiff, Ops);
          cmp_lhs = RID;
          cmp_rhs = getSPIRVInt32Constant(0);
          break;
        default:
          llvm_unreachable("Unexpected signed pointer comparison");
          break;
        }

        Ops.clear();
        Ops << CmpI->getType() << cmp_lhs << cmp_rhs;
        RID = addSPIRVInst(Opcode, Ops);

        setVariablePointersCapabilities(ArgTy->getPointerAddressSpace());
      } else {
        CmpI->print(errs());
        std::string name = I.getParent()->getParent()->getName().str();
        errs() << "\nPointer comparisons not supported prior to SPIR-V 1.4, "
               << "in function " << name << "\n";
        llvm_unreachable("Pointer comparisons not supported");
      }
      break;
    }

    SPIRVOperandVec Ops;
    if (CmpI->getPredicate() == CmpInst::FCMP_ORD ||
        CmpI->getPredicate() == CmpInst::FCMP_UNO) {
      // Implement ordered and unordered comparisons are OpIsNan instructions.
      // Optimize the constants to simplify the resulting code.
      auto lhs = CmpI->getOperand(0);
      auto rhs = CmpI->getOperand(1);
      auto const_lhs = dyn_cast_or_null<Constant>(lhs);
      auto const_rhs = dyn_cast_or_null<Constant>(rhs);
      if ((const_lhs && const_lhs->isNaN()) ||
          (const_rhs && const_rhs->isNaN())) {
        // Result is a constant, false of ordered, true for unordered.
        if (CmpI->getPredicate() == CmpInst::FCMP_ORD) {
          RID = getSPIRVConstant(ConstantInt::getFalse(CmpI->getType()));
        } else {
          RID = getSPIRVConstant(ConstantInt::getTrue(CmpI->getType()));
        }
        break;
      }
      SPIRVID lhs_id;
      SPIRVID rhs_id;
      if (!const_lhs) {
        // Generate OpIsNan for the lhs.
        Ops.clear();
        Ops << CmpI->getType() << lhs;
        lhs_id = addSPIRVInst(spv::OpIsNan, Ops);
      }
      if (!const_rhs) {
        // Generate OpIsNan for the rhs.
        Ops.clear();
        Ops << CmpI->getType() << rhs;
        rhs_id = addSPIRVInst(spv::OpIsNan, Ops);
      }
      if (lhs_id.isValid() && rhs_id.isValid()) {
        // Or the results for the lhs and rhs.
        Ops.clear();
        Ops << CmpI->getType() << lhs_id << rhs_id;
        RID = addSPIRVInst(spv::OpLogicalOr, Ops);
      } else {
        RID = lhs_id.isValid() ? lhs_id : rhs_id;
      }
      if (CmpI->getPredicate() == CmpInst::FCMP_ORD) {
        // For ordered comparisons, invert the intermediate result.
        Ops.clear();
        Ops << CmpI->getType() << RID;
        RID = addSPIRVInst(spv::OpLogicalNot, Ops);
      }
      break;
    } else {
      // Remaining comparisons map directly to SPIR-V opcodes.
      // Ops[0] = Result Type ID
      // Ops[1] = Operand 1 ID
      // Ops[2] = Operand 2 ID
      Ops << CmpI->getType() << CmpI->getOperand(0) << CmpI->getOperand(1);

      spv::Op Opcode = GetSPIRVCmpOpcode(CmpI);
      RID = addSPIRVInst(Opcode, Ops);
    }
    break;
  }
  case Instruction::CondBr:
  case Instruction::UncondBr: {
    // Branch instruction is deferred because it needs label's ID.
    BasicBlock *BrBB = I.getParent();
    if (ContinueBlocks.count(BrBB) || MergeBlocks.count(BrBB)) {
      // Placeholder for Merge operation
      RID = addSPIRVPlaceholder(&I);
    }
    RID = addSPIRVPlaceholder(&I);
    break;
  }
  case Instruction::Switch: {
    I.print(errs());
    llvm_unreachable("Unsupported instruction???");
    break;
  }
  case Instruction::IndirectBr: {
    I.print(errs());
    llvm_unreachable("Unsupported instruction???");
    break;
  }
  case Instruction::PHI: {
    // PHI instruction is deferred because it needs label's ID.
    RID = addSPIRVPlaceholder(&I);
    break;
  }
  case Instruction::Alloca: {
    //
    // Generate OpVariable.
    //
    // Ops[0] : Result Type ID
    // Ops[1] : Storage Class
    SPIRVOperandVec Ops;

    Ops << getSPIRVPointerType(I.getType(),
                               cast<AllocaInst>(&I)->getAllocatedType())
        << spv::StorageClassFunction;

    RID = addSPIRVInst(spv::OpVariable, Ops);
    break;
  }
  case Instruction::Load: {
    LoadInst *LD = cast<LoadInst>(&I);
    //
    // Generate OpLoad.
    //

    if (LD->getType()->isPointerTy()) {
      // Loading a pointer requires variable pointers.
      setVariablePointersCapabilities(GetPointerAddressSpace(LD));
    }

    SPIRVID PointerID = getSPIRVValue(LD->getPointerOperand(), LD->getType());
    // This is a hack to work around what looks like a driver bug.
    // When we're loading from the special variable holding the WorkgroupSize
    // builtin value, use an OpBitWiseAnd of the value's ID rather than
    // generating a load.
    // TODO(dneto): Remove this awful hack once drivers are fixed.
    if (PointerID == WorkgroupSizeVarID) {
      // Generate a bitwise-and of the original value with itself.
      // We should have been able to get away with just an OpCopyObject,
      // but we need something more complex to get past certain driver bugs.
      // This is ridiculous, but necessary.
      // TODO(dneto): Revisit this once drivers fix their bugs.

      SPIRVOperandVec Ops;
      Ops << LD->getType() << WorkgroupSizeValueID << WorkgroupSizeValueID;

      RID = addSPIRVInst(spv::OpBitwiseAnd, Ops);
      break;
    }

    // This is the normal path.  Generate a load.

    // Ops[0] = Result Type ID
    // Ops[1] = Pointer ID
    // Ops[2] ... Ops[n] = Optional Memory Access
    //
    // TODO: Do we need to implement Optional Memory Access???

    auto ptr = LD->getPointerOperand();
    SPIRVID result_type_id;
    if (LD->getType()->isPointerTy()) {
      result_type_id = getSPIRVType(LD->getType());
    } else {
      auto layout = PointerRequiresLayout(GetPointerAddressSpace(ptr));
      result_type_id = getSPIRVType(LD->getType(), layout);
    }
    SPIRVOperandVec Ops;
    Ops << result_type_id << getSPIRVPointerOperand(ptr, LD->getType());

    // Align MemoryOperand helps load vectorization and is required for
    // PhysicalStorageBuffer
    Ops << spv::MemoryAccessAlignedMask;
    Ops << static_cast<uint32_t>(LD->getAlign().value());

    RID = addSPIRVInst(spv::OpLoad, Ops);

    auto no_layout_id = getSPIRVType(LD->getType());
    if (no_layout_id.get() != result_type_id.get()) {
      RID = ChangeLayout(RID, LD->getType(), /* removeLayout = */ true);
    }
    break;
  }
  case Instruction::Store: {
    StoreInst *ST = cast<StoreInst>(&I);
    //
    // Generate OpStore.
    //

    if (ST->getValueOperand()->getType()->isPointerTy()) {
      // Storing a pointer requires variable pointers.
      setVariablePointersCapabilities(
          GetPointerAddressSpace(ST->getValueOperand()));
    }

    SPIRVOperandVec Ops;
    auto ptr = ST->getPointerOperand();
    auto value = ST->getValueOperand();
    auto value_ty = value->getType();
    auto needs_layout = PointerRequiresLayout(GetPointerAddressSpace(ptr));
    if (needs_layout && (value_ty->isArrayTy() || value_ty->isStructTy())) {
      RID = ChangeLayout(getSPIRVValue(value), value_ty,
                         /* removeLayout = */ false);
    }

    // Ops[0] = Pointer ID
    // Ops[1] = Object ID
    // Ops[2] ... Ops[n] = Optional Memory Access (later???)
    //
    // TODO: Do we need to implement Optional Memory Access???
    Ops << getSPIRVPointerOperand(ptr, value_ty);
    if (RID.isValid()) {
      Ops << RID;
    } else {
      Ops << ST->getValueOperand();
    }

    // Align MemoryOperand helps store vectorization and is required for
    // PhysicalStorageBuffer
    Ops << spv::MemoryAccessAlignedMask;
    Ops << static_cast<uint32_t>(ST->getAlign().value());

    RID = addSPIRVInst(spv::OpStore, Ops);
    break;
  }
  case Instruction::AtomicCmpXchg: {
    I.print(errs());
    llvm_unreachable("Unsupported instruction???");
    break;
  }
  case Instruction::AtomicRMW: {
    AtomicRMWInst *AtomicRMW = dyn_cast<AtomicRMWInst>(&I);

    spv::Op opcode;

    switch (AtomicRMW->getOperation()) {
    case llvm::AtomicRMWInst::Add:
      opcode = spv::OpAtomicIAdd;
      break;
    case llvm::AtomicRMWInst::Sub:
      opcode = spv::OpAtomicISub;
      break;
    case llvm::AtomicRMWInst::Xchg:
      opcode = spv::OpAtomicExchange;
      break;
    case llvm::AtomicRMWInst::Min:
      opcode = spv::OpAtomicSMin;
      break;
    case llvm::AtomicRMWInst::Max:
      opcode = spv::OpAtomicSMax;
      break;
    case llvm::AtomicRMWInst::UMin:
      opcode = spv::OpAtomicUMin;
      break;
    case llvm::AtomicRMWInst::UMax:
      opcode = spv::OpAtomicUMax;
      break;
    case llvm::AtomicRMWInst::And:
      opcode = spv::OpAtomicAnd;
      break;
    case llvm::AtomicRMWInst::Or:
      opcode = spv::OpAtomicOr;
      break;
    case llvm::AtomicRMWInst::Xor:
      opcode = spv::OpAtomicXor;
      break;
    default:
      I.print(errs());
      llvm_unreachable("Unsupported instruction???");
    }

    //
    // Generate OpAtomic*.
    //
    SPIRVOperandVec Ops;

    Ops << I.getType()
        << getSPIRVPointerOperand(AtomicRMW->getPointerOperand(),
                                  AtomicRMW->getValOperand()->getType());

    const auto ConstantScopeDevice = getSPIRVInt32Constant(spv::ScopeDevice);
    Ops << ConstantScopeDevice;

    // Vulkan disallows sequentially consistent.
    const auto ConstantMemorySemantics =
        getSPIRVInt32Constant(spv::MemorySemanticsUniformMemoryMask |
                              spv::MemorySemanticsAcquireReleaseMask);
    Ops << ConstantMemorySemantics << AtomicRMW->getValOperand();

    RID = addSPIRVInst(opcode, Ops);
    break;
  }
  case Instruction::Fence: {
    I.print(errs());
    llvm_unreachable("Unsupported instruction???");
    break;
  }
  case Instruction::Call: {
    CallInst *Call = dyn_cast<CallInst>(&I);
    RID = GenerateInstructionFromCall(Call);
    break;
  }
  case Instruction::Ret: {
    unsigned NumOps = I.getNumOperands();
    if (NumOps == 0) {
      //
      // Generate OpReturn.
      //
      RID = addSPIRVInst(spv::OpReturn);
    } else {
      //
      // Generate OpReturnValue.
      //

      // Ops[0] = Return Value ID
      SPIRVOperandVec Ops;

      Ops << I.getOperand(0);

      RID = addSPIRVInst(spv::OpReturnValue, Ops);
      break;
    }
    break;
  }
  }

  // Register Instruction to ValueMap.
  if (RID.isValid()) {
    VMap[&I] = RID;
  }
}

void SPIRVProducerPassImpl::GenerateFuncEpilogue() {
  //
  // Generate OpFunctionEnd
  //
  addSPIRVInst(spv::OpFunctionEnd);
}

bool SPIRVProducerPassImpl::is4xi8vec(Type *Ty) const {
  // Don't specialize <4 x i8> if i8 is generally supported.
  if (clspv::Option::Int8Support())
    return false;

  LLVMContext &Context = Ty->getContext();
  if (auto VecTy = dyn_cast<VectorType>(Ty)) {
    if (VecTy->getElementType() == Type::getInt8Ty(Context) &&
        VecTy->getElementCount().getKnownMinValue() == 4) {
      return true;
    }
  }

  return false;
}

void SPIRVProducerPassImpl::HandleDeferredInstruction() {
  DeferredInstVecType &DeferredInsts = getDeferredInstVec();

  for (size_t i = 0; i < DeferredInsts.size(); ++i) {
    Value *Inst = DeferredInsts[i].first;
    SPIRVInstruction *Placeholder = DeferredInsts[i].second;
    SPIRVOperandVec Operands;

    auto nextDeferred = [&i, &Inst, &DeferredInsts, &Placeholder]() {
      (void)Inst;
      ++i;
      assert(DeferredInsts.size() > i);
      assert(Inst == DeferredInsts[i].first);
      Placeholder = DeferredInsts[i].second;
    };

    if (isa<CondBrInst>(Inst) || isa<UncondBrInst>(Inst)) {
      // Check whether this branch needs to be preceeded by merge instruction.
      BasicBlock *BrBB = dyn_cast<Instruction>(Inst)->getParent();
      if (ContinueBlocks.count(BrBB)) {
        //
        // Generate OpLoopMerge.
        //
        // Ops[0] = Merge Block ID
        // Ops[1] = Continue Target ID
        // Ops[2] = Selection Control
        SPIRVOperandVec Ops;

        Ops << MergeBlocks[BrBB] << ContinueBlocks[BrBB]
            << spv::LoopControlMaskNone;

        replaceSPIRVInst(Placeholder, spv::OpLoopMerge, Ops);

        nextDeferred();

      } else if (MergeBlocks.count(BrBB)) {
        //
        // Generate OpSelectionMerge.
        //
        // Ops[0] = Merge Block ID
        // Ops[1] = Selection Control
        SPIRVOperandVec Ops;

        auto MergeBB = MergeBlocks[BrBB];
        Ops << MergeBB << spv::SelectionControlMaskNone;

        replaceSPIRVInst(Placeholder, spv::OpSelectionMerge, Ops);

        nextDeferred();
      }

      if (auto *CondBr = dyn_cast_or_null<CondBrInst>(Inst)) {
        //
        // Generate OpBranchConditional.
        //
        // Ops[0] = Condition ID
        // Ops[1] = True Label ID
        // Ops[2] = False Label ID
        // Ops[3] ... Ops[n] = Branch weights (Literal Number)
        SPIRVOperandVec Ops;

        Ops << CondBr->getCondition() << CondBr->getSuccessor(0)
            << CondBr->getSuccessor(1);

        replaceSPIRVInst(Placeholder, spv::OpBranchConditional, Ops);

      } else {
        auto *UncondBr = dyn_cast<UncondBrInst>(Inst);
        //
        // Generate OpBranch.
        //
        // Ops[0] = Target Label ID
        SPIRVOperandVec Ops;

        Ops << UncondBr->getSuccessor(0);

        replaceSPIRVInst(Placeholder, spv::OpBranch, Ops);
      }
    } else if (PHINode *PHI = dyn_cast<PHINode>(Inst)) {
      auto PHITy = PHI->getType();
      auto &GlobalConstArgSet = getGlobalConstArgSet();
      if (PHITy->isPointerTy()) {
        for (unsigned j = 0; j < PHI->getNumIncomingValues(); j++) {
          if (GlobalConstArgSet.count(PHI->getIncomingValue(j))) {
            PHITy = PointerType::get(module->getContext(),
                                     AddressSpace::ModuleScopePrivate);
            GlobalConstArgSet.insert(PHI);
            break;
          }
        }
        // OpPhi on pointers requires variable pointers.
        setVariablePointersCapabilities(PHITy->getPointerAddressSpace());
        if (GetStorageClass(PHITy->getPointerAddressSpace()) !=
                spv::StorageClassPhysicalStorageBuffer &&
            !hasVariablePointers() && !selectFromSameObject(PHI)) {
          setVariablePointers();
        }
      }

      //
      // Generate OpPhi.
      //
      // Ops[0] = Result Type ID
      // Ops[1] ... Ops[n] = (Variable ID, Parent ID) pairs
      SPIRVOperandVec Ops;

      Type *TyHint = PHI->getType();
      if (PHITy->isPointerTy()) {
        TyHint = InferType(PHI, module->getContext(), &InferredTypeCache);
        Ops << getSPIRVPointerType(PHITy, TyHint);
      } else {
        Ops << PHITy;
      }

      for (unsigned j = 0; j < PHI->getNumIncomingValues(); j++) {
        Ops << getSPIRVValue(PHI->getIncomingValue(j), TyHint)
            << PHI->getIncomingBlock(j);
      }

      replaceSPIRVInst(Placeholder, spv::OpPhi, Ops);

    } else if (CallInst *Call = dyn_cast<CallInst>(Inst)) {
      Function *Callee = Call->getCalledFunction();
      auto callee_name = Callee->getName();

      if (Builtins::Lookup(Callee) == Builtins::kClspvCompositeConstruct) {
        // Generate an OpCompositeConstruct
        SPIRVOperandVec Ops;

        // The result type.
        Ops << Call->getType();

        for (Use &use : Call->args()) {
          Ops << use.get();
        }

        replaceSPIRVInst(Placeholder, spv::OpCompositeConstruct, Ops);

      } else {
        if (Call->getType()->isPointerTy()) {
          // Functions returning pointers require variable pointers.
          setVariablePointersCapabilities(
              Call->getType()->getPointerAddressSpace());
        }

        //
        // Generate OpFunctionCall.
        //

        // Ops[0] = Result Type ID
        // Ops[1] = Callee Function ID
        // Ops[2] ... Ops[n] = Argument 0, ... , Argument n
        SPIRVOperandVec Ops;

        SPIRVID type_id;
        if (Call->getType()->isPointerTy()) {
          auto *inferred_ty =
              InferType(Call, module->getContext(), &InferredTypeCache);
          type_id = getSPIRVPointerType(Call->getType(), inferred_ty);
        } else {
          type_id = getSPIRVType(Call->getType());
        }
        Ops << type_id;

        SPIRVID CalleeID = getSPIRVValue(Callee);
        if (!CalleeID.isValid()) {
          std::string err_msg =
              "error: undefined reference to '" + callee_name.str() + "'\n";
          if (OutputLog) {
            OutputLog->append(err_msg);
          } else {
            errs() << err_msg;
          }
          HasError = true;
          return;
        }

        Ops << CalleeID;

        FunctionType *CalleeFTy = cast<FunctionType>(Call->getFunctionType());
        for (unsigned j = 0; j < CalleeFTy->getNumParams(); j++) {
          auto *operand = Call->getOperand(j);
          auto *operand_type = operand->getType();
          auto *inferred_ty =
              InferType(operand, module->getContext(), &InferredTypeCache);
          StructType *struct_ty = dyn_cast_or_null<StructType>(inferred_ty);
          // Images and samplers can be passed as function parameters without
          // variable pointers.
          if (operand_type->isPointerTy() && !IsImageType(struct_ty) &&
              !IsSamplerType(struct_ty)) {
            auto sc = GetStorageClass(GetPointerAddressSpace(operand));
            if (sc == spv::StorageClassStorageBuffer) {
              // Passing SSBO by reference requires variable pointers storage
              // buffer.
              setVariablePointersStorageBuffer();
            } else if (sc == spv::StorageClassWorkgroup) {
              // Workgroup references require variable pointers if they are not
              // memory object declarations.
              if (auto *operand_call = dyn_cast<CallInst>(operand)) {
                // Workgroup accessor represents a variable reference.
                if (Builtins::Lookup(operand_call->getCalledFunction()) !=
                    Builtins::kClspvLocal)
                  setVariablePointers();
              } else {
                // Arguments are function parameters.
                if (!isa<Argument>(operand))
                  setVariablePointers();
              }
            }
          }
          Ops << operand;
        }

        replaceSPIRVInst(Placeholder, spv::OpFunctionCall, Ops);
      }
    }
  }
}

void SPIRVProducerPassImpl::HandleDeferredDecorations() {
  if (getTypesNeedingArrayStride().empty()) {
    return;
  }

  // Insert ArrayStride decorations on pointer types, due to OpPtrAccessChain
  // instructions we generated earlier.
  DenseSet<uint32_t> seen;
  for (auto stride_type : getTypesNeedingArrayStride()) {
    auto stride = stride_type.stride;
    auto id = stride_type.id;
    assert(id.isValid() && stride != 0);
    if (!seen.insert(id.get()).second)
      continue;

    // Ops[0] = Target ID
    // Ops[1] = Decoration (ArrayStride)
    // Ops[2] = Stride number (Literal Number)
    SPIRVOperandVec Ops;
    // if stride is 1 automatically set it to 4 to avoid stride issues.
    Ops << id << spv::DecorationArrayStride << getCorrectedStride(stride);

    addSPIRVInst<kAnnotations>(spv::OpDecorate, Ops);
  }
}

void SPIRVProducerPassImpl::WriteOneWord(uint32_t Word) {
  binaryOut->write(reinterpret_cast<const char *>(&Word), sizeof(uint32_t));
}

void SPIRVProducerPassImpl::WriteResultID(const SPIRVInstruction &Inst) {
  WriteOneWord(Inst.getResultID().get());
}

void SPIRVProducerPassImpl::WriteWordCountAndOpcode(
    const SPIRVInstruction &Inst) {
  // High 16 bit : Word Count
  // Low 16 bit  : Opcode
  uint32_t Word = Inst.getOpcode();
  const uint32_t count = Inst.getWordCount();
  if (count > 65535) {
    errs() << "Word count limit of 65535 exceeded: " << count << "\n";
    llvm_unreachable("Word count too high");
  }
  Word |= Inst.getWordCount() << 16;
  WriteOneWord(Word);
}

void SPIRVProducerPassImpl::WriteOperand(const SPIRVOperand &Op) {
  SPIRVOperandType OpTy = Op.getType();
  switch (OpTy) {
  default: {
    llvm_unreachable("Unsupported SPIRV Operand Type???");
    break;
  }
  case SPIRVOperandType::NUMBERID: {
    WriteOneWord(Op.getNumID());
    break;
  }
  case SPIRVOperandType::LITERAL_STRING: {
    std::string Str = Op.getLiteralStr();
    const char *Data = Str.c_str();
    size_t WordSize = Str.size() / 4;
    for (unsigned Idx = 0; Idx < WordSize; Idx++) {
      WriteOneWord(*reinterpret_cast<const uint32_t *>(&Data[4 * Idx]));
    }

    uint32_t Remainder = Str.size() % 4;
    uint32_t LastWord = 0;
    if (Remainder) {
      for (unsigned Idx = 0; Idx < Remainder; Idx++) {
        LastWord |= Data[4 * WordSize + Idx] << 8 * Idx;
      }
    }

    WriteOneWord(LastWord);
    break;
  }
  case SPIRVOperandType::LITERAL_WORD: {
    WriteOneWord(Op.getLiteralNum()[0]);
    break;
  }
  case SPIRVOperandType::LITERAL_DWORD: {
    WriteOneWord(Op.getLiteralNum()[0]);
    WriteOneWord(Op.getLiteralNum()[1]);
    break;
  }
  }
}

void SPIRVProducerPassImpl::WriteSPIRVBinary() {
  for (int i = 0; i < kSectionCount; ++i) {
    WriteSPIRVBinary(SPIRVSections[i]);
  }
}

void SPIRVProducerPassImpl::WriteSPIRVBinary(
    SPIRVInstructionList &SPIRVInstList) {
  for (const auto &Inst : SPIRVInstList) {
    const auto &Ops = Inst.getOperands();
    spv::Op Opcode = static_cast<spv::Op>(Inst.getOpcode());

    switch (Opcode) {
    default: {
      errs() << "Unsupported SPIR-V instruction opcode " << int(Opcode) << "\n";
      llvm_unreachable("Unsupported SPIRV instruction");
      break;
    }
    case spv::OpName:
    case spv::OpLine:
    case spv::OpNoLine:
    case spv::OpUnreachable:
    case spv::OpCapability:
    case spv::OpExtension:
    case spv::OpMemoryModel:
    case spv::OpEntryPoint:
    case spv::OpExecutionMode:
    case spv::OpSource:
    case spv::OpDecorate:
    case spv::OpMemberDecorate:
    case spv::OpBranch:
    case spv::OpBranchConditional:
    case spv::OpSelectionMerge:
    case spv::OpLoopMerge:
    case spv::OpStore:
    case spv::OpImageWrite:
    case spv::OpReturnValue:
    case spv::OpControlBarrier:
    case spv::OpMemoryBarrier:
    case spv::OpReturn:
    case spv::OpFunctionEnd:
    case spv::OpCopyMemory:
    case spv::OpCopyMemorySized:
    case spv::OpAtomicStore: {
      WriteWordCountAndOpcode(Inst);
      for (uint32_t i = 0; i < Ops.size(); i++) {
        WriteOperand(Ops[i]);
      }
      break;
    }
    case spv::OpTypeBool:
    case spv::OpTypeVoid:
    case spv::OpTypeSampler:
    case spv::OpLabel:
    case spv::OpExtInstImport:
    case spv::OpTypePointer:
    case spv::OpTypeRuntimeArray:
    case spv::OpTypeStruct:
    case spv::OpTypeImage:
    case spv::OpTypeSampledImage:
    case spv::OpTypeInt:
    case spv::OpTypeFloat:
    case spv::OpTypeArray:
    case spv::OpTypeVector:
    case spv::OpTypeFunction:
    case spv::OpTypeUntypedPointerKHR:
    case spv::OpString: {
      WriteWordCountAndOpcode(Inst);
      WriteResultID(Inst);
      for (uint32_t i = 0; i < Ops.size(); i++) {
        WriteOperand(Ops[i]);
      }
      break;
    }
    case spv::OpFmaKHR:
    case spv::OpFunction:
    case spv::OpFunctionParameter:
    case spv::OpAccessChain:
    case spv::OpUntypedAccessChainKHR:
    case spv::OpPtrAccessChain:
    case spv::OpUntypedPtrAccessChainKHR:
    case spv::OpInBoundsAccessChain:
    case spv::OpUntypedInBoundsAccessChainKHR:
    case spv::OpUConvert:
    case spv::OpSConvert:
    case spv::OpConvertFToU:
    case spv::OpConvertFToS:
    case spv::OpConvertUToF:
    case spv::OpConvertSToF:
    case spv::OpFConvert:
    case spv::OpConvertPtrToU:
    case spv::OpConvertUToPtr:
    case spv::OpBitcast:
    case spv::OpBitReverse:
    case spv::OpFNegate:
    case spv::OpIAdd:
    case spv::OpIAddCarry:
    case spv::OpFAdd:
    case spv::OpISub:
    case spv::OpISubBorrow:
    case spv::OpFSub:
    case spv::OpIMul:
    case spv::OpFMul:
    case spv::OpUDiv:
    case spv::OpSDiv:
    case spv::OpFDiv:
    case spv::OpUMod:
    case spv::OpSRem:
    case spv::OpFRem:
    case spv::OpUMulExtended:
    case spv::OpSMulExtended:
    case spv::OpBitwiseOr:
    case spv::OpBitwiseXor:
    case spv::OpBitwiseAnd:
    case spv::OpNot:
    case spv::OpShiftLeftLogical:
    case spv::OpShiftRightLogical:
    case spv::OpShiftRightArithmetic:
    case spv::OpBitCount:
    case spv::OpCompositeConstruct:
    case spv::OpCompositeExtract:
    case spv::OpVectorExtractDynamic:
    case spv::OpCompositeInsert:
    case spv::OpCopyLogical:
    case spv::OpCopyObject:
    case spv::OpVectorInsertDynamic:
    case spv::OpVectorShuffle:
    case spv::OpIEqual:
    case spv::OpINotEqual:
    case spv::OpPtrEqual:
    case spv::OpPtrNotEqual:
    case spv::OpPtrDiff:
    case spv::OpUGreaterThan:
    case spv::OpUGreaterThanEqual:
    case spv::OpULessThan:
    case spv::OpULessThanEqual:
    case spv::OpSGreaterThan:
    case spv::OpSGreaterThanEqual:
    case spv::OpSLessThan:
    case spv::OpSLessThanEqual:
    case spv::OpFOrdEqual:
    case spv::OpFOrdGreaterThan:
    case spv::OpFOrdGreaterThanEqual:
    case spv::OpFOrdLessThan:
    case spv::OpFOrdLessThanEqual:
    case spv::OpFOrdNotEqual:
    case spv::OpFUnordEqual:
    case spv::OpFUnordGreaterThan:
    case spv::OpFUnordGreaterThanEqual:
    case spv::OpFUnordLessThan:
    case spv::OpFUnordLessThanEqual:
    case spv::OpFUnordNotEqual:
    case spv::OpExtInst:
    case spv::OpIsInf:
    case spv::OpIsNan:
    case spv::OpAny:
    case spv::OpAll:
    case spv::OpUndef:
    case spv::OpConstantNull:
    case spv::OpLogicalOr:
    case spv::OpLogicalAnd:
    case spv::OpLogicalNot:
    case spv::OpLogicalNotEqual:
    case spv::OpConstantComposite:
    case spv::OpSpecConstantComposite:
    case spv::OpConstantTrue:
    case spv::OpConstantFalse:
    case spv::OpConstant:
    case spv::OpSpecConstant:
    case spv::OpVariable:
    case spv::OpUntypedVariableKHR:
    case spv::OpFunctionCall:
    case spv::OpSampledImage:
    case spv::OpImageFetch:
    case spv::OpImageRead:
    case spv::OpImageSampleExplicitLod:
    case spv::OpImageQuerySize:
    case spv::OpImageQuerySizeLod:
    case spv::OpSelect:
    case spv::OpPhi:
    case spv::OpLoad:
    case spv::OpAtomicLoad:
    case spv::OpAtomicIAdd:
    case spv::OpAtomicISub:
    case spv::OpAtomicExchange:
    case spv::OpAtomicIIncrement:
    case spv::OpAtomicIDecrement:
    case spv::OpAtomicCompareExchange:
    case spv::OpAtomicUMin:
    case spv::OpAtomicSMin:
    case spv::OpAtomicUMax:
    case spv::OpAtomicSMax:
    case spv::OpAtomicAnd:
    case spv::OpAtomicOr:
    case spv::OpAtomicXor:
    case spv::OpDot:
    case spv::OpUDot:
    case spv::OpSDot:
    case spv::OpSUDot:
    case spv::OpUDotAccSat:
    case spv::OpSDotAccSat:
    case spv::OpSUDotAccSat:
    case spv::OpGroupNonUniformAll:
    case spv::OpGroupNonUniformAny:
    case spv::OpGroupNonUniformAllEqual:
    case spv::OpGroupNonUniformBallot:
    case spv::OpGroupNonUniformInverseBallot:
    case spv::OpGroupNonUniformBallotBitExtract:
    case spv::OpGroupNonUniformBallotBitCount:
    case spv::OpGroupNonUniformBallotFindLSB:
    case spv::OpGroupNonUniformBallotFindMSB:
    case spv::OpGroupNonUniformBroadcast:
    case spv::OpGroupNonUniformBroadcastFirst:
    case spv::OpGroupNonUniformElect:
    case spv::OpGroupNonUniformIAdd:
    case spv::OpGroupNonUniformFAdd:
    case spv::OpGroupNonUniformSMin:
    case spv::OpGroupNonUniformUMin:
    case spv::OpGroupNonUniformFMin:
    case spv::OpGroupNonUniformSMax:
    case spv::OpGroupNonUniformUMax:
    case spv::OpGroupNonUniformFMax:
    case spv::OpGroupNonUniformShuffle:
    case spv::OpGroupNonUniformShuffleXor:
    case spv::OpGroupNonUniformRotateKHR: {
      WriteWordCountAndOpcode(Inst);
      WriteOperand(Ops[0]);
      WriteResultID(Inst);
      for (uint32_t i = 1; i < Ops.size(); i++) {
        WriteOperand(Ops[i]);
      }
      break;
    }
    }
  }
}

bool SPIRVProducerPassImpl::IsTypeNullable(const Type *type) const {
  switch (type->getTypeID()) {
  case Type::HalfTyID:
  case Type::FloatTyID:
  case Type::DoubleTyID:
  case Type::IntegerTyID:
  case Type::FixedVectorTyID:
    return true;
  case Type::PointerTyID: {
    // Samplers and images should not be nulled, but we lack that information
    // here. That said, an undef image/sampler is likely already problematic
    // (e.g. due to a phi).
    return true;
  }
  case Type::ArrayTyID:
    return IsTypeNullable(type->getArrayElementType());
  case Type::StructTyID: {
    const StructType *struct_type = cast<StructType>(type);
    // Images and samplers are not nullable.
    if (struct_type->isOpaque())
      return false;
    for (const auto element : struct_type->elements()) {
      if (!IsTypeNullable(element))
        return false;
    }
    return true;
  }
  default:
    return false;
  }
}

void SPIRVProducerPassImpl::PopulateUBOTypeMaps() {
  if (auto *offsets_md =
          module->getNamedMetadata(clspv::RemappedTypeOffsetMetadataName())) {
    // Metdata is stored as key-value pair operands. The first element of each
    // operand is the type and the second is a vector of offsets.
    for (const auto *operand : offsets_md->operands()) {
      const auto *pair = cast<MDTuple>(operand);
      auto *type =
          cast<ConstantAsMetadata>(pair->getOperand(0))->getValue()->getType();
      const auto *offset_vector = cast<MDTuple>(pair->getOperand(1));
      std::vector<uint32_t> offsets;
      for (const Metadata *offset_md : offset_vector->operands()) {
        const auto *constant_md = cast<ConstantAsMetadata>(offset_md);
        offsets.push_back(static_cast<uint32_t>(
            cast<ConstantInt>(constant_md->getValue())->getZExtValue()));
      }
      RemappedUBOTypeOffsets.insert(std::make_pair(type, offsets));
    }
  }

  if (auto *sizes_md =
          module->getNamedMetadata(clspv::RemappedTypeSizesMetadataName())) {
    // Metadata is stored as key-value pair operands. The first element of each
    // operand is the type and the second is a triple of sizes: type size in
    // bits, store size and alloc size.
    for (const auto *operand : sizes_md->operands()) {
      const auto *pair = cast<MDTuple>(operand);
      auto *type =
          cast<ConstantAsMetadata>(pair->getOperand(0))->getValue()->getType();
      const auto *size_triple = cast<MDTuple>(pair->getOperand(1));
      uint64_t type_size_in_bits =
          cast<ConstantInt>(
              cast<ConstantAsMetadata>(size_triple->getOperand(0))->getValue())
              ->getZExtValue();
      uint64_t type_store_size =
          cast<ConstantInt>(
              cast<ConstantAsMetadata>(size_triple->getOperand(1))->getValue())
              ->getZExtValue();
      uint64_t type_alloc_size =
          cast<ConstantInt>(
              cast<ConstantAsMetadata>(size_triple->getOperand(2))->getValue())
              ->getZExtValue();
      RemappedUBOTypeSizes.insert(std::make_pair(
          type, std::make_tuple(type_size_in_bits, type_store_size,
                                type_alloc_size)));
    }
  }
}

uint64_t SPIRVProducerPassImpl::GetTypeSizeInBits(Type *type,
                                                  const DataLayout &DL) {
  auto iter = RemappedUBOTypeSizes.find(type);
  if (iter != RemappedUBOTypeSizes.end()) {
    return std::get<0>(iter->second);
  }

  return DL.getTypeSizeInBits(type);
}

uint64_t SPIRVProducerPassImpl::GetTypeAllocSize(Type *type,
                                                 const DataLayout &DL) {
  auto iter = RemappedUBOTypeSizes.find(type);
  if (iter != RemappedUBOTypeSizes.end()) {
    return std::get<2>(iter->second);
  }

  return DL.getTypeAllocSize(type);
}

uint32_t SPIRVProducerPassImpl::GetExplicitLayoutStructMemberOffset(
    StructType *type, unsigned member, const DataLayout &DL) {
  const auto StructLayout = DL.getStructLayout(type);
  // Search for the correct offsets if this type was remapped.
  std::vector<uint32_t> *offsets = nullptr;
  auto iter = RemappedUBOTypeOffsets.find(type);
  if (iter != RemappedUBOTypeOffsets.end()) {
    offsets = &iter->second;
  }
  auto ByteOffset =
      static_cast<uint32_t>(StructLayout->getElementOffset(member));
  if (offsets) {
    ByteOffset = (*offsets)[member];
  }

  return ByteOffset;
}

void SPIRVProducerPassImpl::setVariablePointersCapabilities(
    unsigned address_space) {
  if (GetStorageClass(address_space) ==
      spv::StorageClassPhysicalStorageBuffer) {
    return;
  }
  if (GetStorageClass(address_space) == spv::StorageClassStorageBuffer) {
    setVariablePointersStorageBuffer();
  } else {
    setVariablePointers();
  }
}

Value *SPIRVProducerPassImpl::GetBasePointer(Value *v) {
  if (auto *gep = dyn_cast<GetElementPtrInst>(v)) {
    return GetBasePointer(gep->getPointerOperand());
  }

  // Conservatively return |v|.
  return v;
}

bool SPIRVProducerPassImpl::sameResource(Value *lhs, Value *rhs) const {
  if (auto *lhs_call = dyn_cast<CallInst>(lhs)) {
    if (auto *rhs_call = dyn_cast<CallInst>(rhs)) {
      const auto &lhs_func_info =
          Builtins::Lookup(lhs_call->getCalledFunction());
      const auto &rhs_func_info =
          Builtins::Lookup(rhs_call->getCalledFunction());
      if (lhs_func_info.getType() == Builtins::kClspvResource &&
          rhs_func_info.getType() == Builtins::kClspvResource) {
        // For resource accessors, match descriptor set and binding.
        if (lhs_call->getOperand(ClspvOperand::kResourceDescriptorSet) ==
                rhs_call->getOperand(ClspvOperand::kResourceDescriptorSet) &&
            lhs_call->getOperand(ClspvOperand::kResourceBinding) ==
                rhs_call->getOperand(ClspvOperand::kResourceBinding))
          return true;
      } else if (lhs_func_info.getType() == Builtins::kClspvLocal &&
                 rhs_func_info.getType() == Builtins::kClspvLocal) {
        // For workgroup resources, match spec id.
        if (lhs_call->getOperand(ClspvOperand::kWorkgroupSpecId) ==
            rhs_call->getOperand(ClspvOperand::kWorkgroupSpecId))
          return true;
      }
    }
  }

  return false;
}

bool SPIRVProducerPassImpl::selectFromSameObject(Instruction *inst) {
  assert(inst->getType()->isPointerTy());
  auto sc = GetStorageClass(inst->getType()->getPointerAddressSpace());
  if (sc == spv::StorageClassPhysicalStorageBuffer) {
    return true;
  }
  assert(sc == spv::StorageClassStorageBuffer);
  const bool hack_undef = clspv::Option::HackUndef();
  if (auto *select = dyn_cast<SelectInst>(inst)) {
    auto *true_base = GetBasePointer(select->getTrueValue());
    auto *false_base = GetBasePointer(select->getFalseValue());

    if (true_base == false_base)
      return true;

    // If either the true or false operand is a null, then we satisfy the same
    // object constraint.
    if (auto *true_cst = dyn_cast<Constant>(true_base)) {
      if (true_cst->isNullValue() || (hack_undef && isa<UndefValue>(true_base)))
        return true;
    }

    if (auto *false_cst = dyn_cast<Constant>(false_base)) {
      if (false_cst->isNullValue() ||
          (hack_undef && isa<UndefValue>(false_base)))
        return true;
    }

    if (sameResource(true_base, false_base))
      return true;
  } else if (auto *phi = dyn_cast<PHINode>(inst)) {
    Value *value = nullptr;
    bool ok = true;
    for (unsigned i = 0; ok && i != phi->getNumIncomingValues(); ++i) {
      auto *base = GetBasePointer(phi->getIncomingValue(i));
      // Null values satisfy the constraint of selecting of selecting from the
      // same object.
      if (!value) {
        if (auto *cst = dyn_cast<Constant>(base)) {
          if (!cst->isNullValue() && !(hack_undef && isa<UndefValue>(base)))
            value = base;
        } else {
          value = base;
        }
      } else if (base != value) {
        if (auto *base_cst = dyn_cast<Constant>(base)) {
          if (base_cst->isNullValue() || (hack_undef && isa<UndefValue>(base)))
            continue;
        }

        if (sameResource(value, base))
          continue;

        // Values don't represent the same base.
        ok = false;
      }
    }

    return ok;
  }

  // Conservatively return false.
  return false;
}

bool SPIRVProducerPassImpl::isPointerUniform(Value *ptr) {
  if (!ptr->getType()->isPointerTy()) {
    return true;
  }
  if (GetStorageClass(ptr->getType()->getPointerAddressSpace()) !=
      spv::StorageClassStorageBuffer) {
    return true;
  }

  while (auto gep = dyn_cast<GetElementPtrInst>(ptr)) {
    ptr = gep->getPointerOperand();
  }

  if (NonUniformPointers.count(ptr) > 0) {
    return false;
  }

  bool uniformPointer = true;
  if (auto inst = dyn_cast<PHINode>(ptr)) {
    uniformPointer = selectFromSameObject(inst);
  } else if (auto inst = dyn_cast<SelectInst>(ptr)) {
    uniformPointer = selectFromSameObject(inst);
  }
  if (!uniformPointer) {
    NonUniformPointers.insert(ptr);
  }
  return uniformPointer;
}

bool SPIRVProducerPassImpl::CalledWithCoherentResource(Argument &Arg) {
  if (!Arg.getType()->isPointerTy() ||
      Arg.getType()->getPointerAddressSpace() != clspv::AddressSpace::Global) {
    // Only SSBOs need to be annotated as coherent.
    return false;
  }

  DenseSet<Value *> visited;
  std::vector<Value *> stack;
  for (auto *U : Arg.getParent()->users()) {
    if (auto *call = dyn_cast<CallInst>(U)) {
      stack.push_back(call->getOperand(Arg.getArgNo()));
    }
  }

  while (!stack.empty()) {
    Value *v = stack.back();
    stack.pop_back();

    if (!visited.insert(v).second)
      continue;

    auto *resource_call = dyn_cast<CallInst>(v);
    if (resource_call &&
        Builtins::Lookup(resource_call->getCalledFunction()).getType() ==
            Builtins::kClspvResource) {
      // If this is a resource accessor function, check if the coherent operand
      // is set.
      const auto coherent =
          unsigned(dyn_cast<ConstantInt>(resource_call->getArgOperand(
                                             ClspvOperand::kResourceCoherent))
                       ->getZExtValue());
      if (coherent == 1)
        return true;
    } else if (auto *arg = dyn_cast<Argument>(v)) {
      // If this is a function argument, trace through its callers.
      for (auto U : arg->getParent()->users()) {
        if (auto *call = dyn_cast<CallInst>(U)) {
          stack.push_back(call->getOperand(arg->getArgNo()));
        }
      }
    } else if (auto *user = dyn_cast<User>(v)) {
      // If this is a user, traverse all operands that could lead to resource
      // variables.
      for (unsigned i = 0; i != user->getNumOperands(); ++i) {
        Value *operand = user->getOperand(i);
        if (operand->getType()->isPointerTy() &&
            operand->getType()->getPointerAddressSpace() ==
                clspv::AddressSpace::Global) {
          stack.push_back(operand);
        }
      }
    }
  }

  // No coherent resource variables encountered.
  return false;
}

void SPIRVProducerPassImpl::PopulateStructuredCFGMaps() {
  // First, track loop merges and continues.
  DenseSet<BasicBlock *> LoopMergesAndContinues;
  for (auto &F : *module) {
    if (F.isDeclaration())
      continue;

    auto &FAM = MAM->getResult<FunctionAnalysisManagerModuleProxy>(*module)
                    .getManager();
    auto &DT = FAM.getResult<DominatorTreeAnalysis>(F);
    auto &LI = FAM.getResult<LoopAnalysis>(F);
    std::deque<BasicBlock *> order;
    DenseSet<BasicBlock *> visited;
    clspv::ComputeStructuredOrder(&*F.begin(), &DT, LI, &order, &visited);

    for (auto BB : order) {
      auto terminator = BB->getTerminator();
      if (LI.isLoopHeader(BB)) {
        auto L = LI.getLoopFor(BB);
        BasicBlock *ContinueBB = nullptr;
        BasicBlock *MergeBB = nullptr;

        MergeBB = L->getExitBlock();
        if (!MergeBB) {
          // StructurizeCFG pass converts CFG into triangle shape and the cfg
          // has regions with single entry/exit. As a result, loop should not
          // have multiple exits.
          llvm_unreachable("Loop has multiple exits???");
        }

        if (L->isLoopLatch(BB)) {
          ContinueBB = BB;
        } else {
          // From SPIR-V spec 2.11, Continue Target must dominate that back-edge
          // block.
          BasicBlock *Header = L->getHeader();
          BasicBlock *Latch = L->getLoopLatch();
          for (auto *loop_block : L->blocks()) {
            if (loop_block == Header) {
              continue;
            }

            // Check whether block dominates block with back-edge.
            // The loop latch is the single block with a back-edge. If it was
            // possible, StructurizeCFG made the loop conform to this
            // requirement, otherwise |Latch| is a nullptr.
            if (DT.dominates(loop_block, Latch)) {
              ContinueBB = loop_block;
            }
          }

          if (!ContinueBB) {
            llvm_unreachable("Wrong continue block from loop");
          }
        }

        // Record the continue and merge blocks.
        MergeBlocks[BB] = MergeBB;
        ContinueBlocks[BB] = ContinueBB;
        LoopMergesAndContinues.insert(MergeBB);
        LoopMergesAndContinues.insert(ContinueBB);
      } else if (auto *cond_branch = dyn_cast_or_null<CondBrInst>(terminator)) {
        auto L = LI.getLoopFor(BB);
        bool HasBackedge = false;
        while (L && !HasBackedge) {
          if (L->isLoopLatch(BB)) {
            HasBackedge = true;
          }
          L = L->getParentLoop();
        }

        if (!HasBackedge) {
          // Only need a merge if the branch doesn't include a loop break or
          // continue.
          auto true_bb = cond_branch->getSuccessor(0);
          auto false_bb = cond_branch->getSuccessor(1);
          if (!LoopMergesAndContinues.count(true_bb) &&
              !LoopMergesAndContinues.count(false_bb)) {
            // StructurizeCFG pass already manipulated CFG. Just use false block
            // of branch instruction as merge block.
            MergeBlocks[BB] = false_bb;
          }
        }
      }
    }
  }
}

SPIRVID SPIRVProducerPassImpl::getReflectionImport() {
  if (!ReflectionID.isValid()) {
    if (SpvVersion() < clspv::Option::SPIRVVersion::SPIRV_1_6) {
      addSPIRVInst<kExtensions>(spv::OpExtension, "SPV_KHR_non_semantic_info");
    }
    ReflectionID = addSPIRVInst<kImports>(spv::OpExtInstImport,
                                          "NonSemantic.ClspvReflection.5");
  }
  return ReflectionID;
}

void SPIRVProducerPassImpl::GenerateReflection() {
  GenerateKernelReflection();
  GeneratePrintfReflection();
  GeneratePushConstantReflection();
  GenerateSpecConstantReflection();
  GenerateWorkgroupSizeReflection();
}

void SPIRVProducerPassImpl::GenerateWorkgroupSizeReflection() {
  auto import_id = getReflectionImport();
  auto void_id = getSPIRVType(Type::getVoidTy(module->getContext()));
  for (auto workgroup_var : WorkgroupVariableSizeMap) {
    SPIRVOperandVec Ops;
    Ops << void_id << import_id << reflection::ExtInstWorkgroupVariableSize
        << workgroup_var.getFirst()
        << getSPIRVInt32Constant(workgroup_var.getSecond());
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
}

void SPIRVProducerPassImpl::GeneratePushConstantReflection() {
  if (auto GV = module->getGlobalVariable(clspv::PushConstantsVariableName())) {
    auto const &DL = module->getDataLayout();
    auto MD = GV->getMetadata(clspv::PushConstantsMetadataName());
    auto STy = cast<StructType>(GV->getValueType());

    for (unsigned i = 0; i < STy->getNumElements(); i++) {
      auto pc = static_cast<clspv::PushConstant>(
          mdconst::extract<ConstantInt>(MD->getOperand(i))->getZExtValue());
      if (pc == PushConstant::KernelArgument ||
          pc == PushConstant::ImageMetadata ||
          pc == PushConstant::ModuleConstantsPointer)
        continue;

      auto memberType = STy->getElementType(i);
      auto offset = GetExplicitLayoutStructMemberOffset(STy, i, DL);
#ifndef NDEBUG
      unsigned previousOffset = 0;
      if (i > 0) {
        previousOffset = GetExplicitLayoutStructMemberOffset(STy, i - 1, DL);
      }
      assert(isValidExplicitLayout(*module, STy, i,
                                   spv::StorageClassPushConstant, offset,
                                   previousOffset));
#endif

      reflection::ExtInst pc_inst = reflection::ExtInstMax;
      switch (pc) {
      case PushConstant::GlobalOffset:
        pc_inst = reflection::ExtInstPushConstantGlobalOffset;
        break;
      case PushConstant::EnqueuedLocalSize:
        pc_inst = reflection::ExtInstPushConstantEnqueuedLocalSize;
        break;
      case PushConstant::GlobalSize:
        pc_inst = reflection::ExtInstPushConstantGlobalSize;
        break;
      case PushConstant::RegionOffset:
        pc_inst = reflection::ExtInstPushConstantRegionOffset;
        break;
      case PushConstant::NumWorkgroups:
        pc_inst = reflection::ExtInstPushConstantNumWorkgroups;
        break;
      case PushConstant::RegionGroupOffset:
        pc_inst = reflection::ExtInstPushConstantRegionGroupOffset;
        break;
      case PushConstant::PrintfBufferPointer:
        pc_inst = reflection::ExtInstPrintfBufferPointerPushConstant;
        break;
      default:
        llvm_unreachable("Unhandled push constant");
        break;
      }

      auto import_id = getReflectionImport();
      auto size = static_cast<uint32_t>(GetTypeSizeInBits(memberType, DL)) / 8;
      SPIRVOperandVec Ops;
      Ops << getSPIRVType(Type::getVoidTy(module->getContext())) << import_id
          << pc_inst << getSPIRVInt32Constant(offset)
          << getSPIRVInt32Constant(size);

      if (pc == PushConstant::PrintfBufferPointer) {
        Ops << getSPIRVInt32Constant(clspv::Option::PrintfBufferSize());
      }
      addSPIRVInst(spv::OpExtInst, Ops);
    }
  }
}

void SPIRVProducerPassImpl::GenerateSpecConstantReflection() {
  const uint32_t kMax = std::numeric_limits<uint32_t>::max();
  uint32_t wgsize_id[3] = {kMax, kMax, kMax};
  uint32_t global_offset_id[3] = {kMax, kMax, kMax};
  uint32_t work_dim_id = kMax;
  uint32_t subgroup_max_size_id = kMax;
  for (auto pair : clspv::GetSpecConstants(module.get())) {
    auto kind = pair.first;
    auto id = pair.second;

    // Local memory size is only used for kernel arguments.
    if (kind == SpecConstant::kLocalMemorySize)
      continue;

    switch (kind) {
    case SpecConstant::kWorkgroupSizeX:
      wgsize_id[0] = id;
      break;
    case SpecConstant::kWorkgroupSizeY:
      wgsize_id[1] = id;
      break;
    case SpecConstant::kWorkgroupSizeZ:
      wgsize_id[2] = id;
      break;
    case SpecConstant::kGlobalOffsetX:
      global_offset_id[0] = id;
      break;
    case SpecConstant::kGlobalOffsetY:
      global_offset_id[1] = id;
      break;
    case SpecConstant::kGlobalOffsetZ:
      global_offset_id[2] = id;
      break;
    case SpecConstant::kWorkDim:
      work_dim_id = id;
      break;
    case SpecConstant::kSubgroupMaxSize:
      subgroup_max_size_id = id;
      break;
    default:
      llvm_unreachable("Unhandled spec constant");
    }
  }

  auto import_id = getReflectionImport();
  auto void_id = getSPIRVType(Type::getVoidTy(module->getContext()));
  SPIRVOperandVec Ops;
  if (wgsize_id[0] != kMax) {
    assert(wgsize_id[1] != kMax);
    assert(wgsize_id[2] != kMax);
    Ops.clear();
    Ops << void_id << import_id << reflection::ExtInstSpecConstantWorkgroupSize
        << getSPIRVInt32Constant(wgsize_id[0])
        << getSPIRVInt32Constant(wgsize_id[1])
        << getSPIRVInt32Constant(wgsize_id[2]);
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
  if (global_offset_id[0] != kMax) {
    assert(global_offset_id[1] != kMax);
    assert(global_offset_id[2] != kMax);
    Ops.clear();
    Ops << void_id << import_id << reflection::ExtInstSpecConstantGlobalOffset
        << getSPIRVInt32Constant(global_offset_id[0])
        << getSPIRVInt32Constant(global_offset_id[1])
        << getSPIRVInt32Constant(global_offset_id[2]);
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
  if (work_dim_id != kMax) {
    Ops.clear();
    Ops << void_id << import_id << reflection::ExtInstSpecConstantWorkDim
        << getSPIRVInt32Constant(work_dim_id);
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
  if (subgroup_max_size_id != kMax) {
    Ops.clear();
    Ops << void_id << import_id
        << reflection::ExtInstSpecConstantSubgroupMaxSize
        << getSPIRVInt32Constant(subgroup_max_size_id);
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
}

void SPIRVProducerPassImpl::GenerateKernelReflection() {
  const auto &DL = module->getDataLayout();
  auto import_id = getReflectionImport();
  auto void_id = getSPIRVType(Type::getVoidTy(module->getContext()));

  for (auto &F : *module) {
    if (F.isDeclaration() || F.getCallingConv() != CallingConv::SPIR_KERNEL) {
      continue;
    }

    // OpString for the kernel name.
    auto kernel_name =
        addSPIRVInst<kDebug>(spv::OpString, F.getName().str().c_str());

    // If we've clustered POD arguments, then argument details are in metadata.
    // If an argument maps to a resource variable, then get descriptor set and
    // binding from the resource variable.  Other info comes from the metadata.
    const auto *arg_map = F.getMetadata(clspv::KernelArgMapMetadataName());
    uint32_t num_args = 0;
    if (arg_map) {
      num_args = arg_map->getNumOperands();
    } else {
      num_args = F.getFunctionType()->getNumParams();
    }

    uint32_t kernel_flags = reflection::ExtKernelPropertyFlags::None;
    if (module->getNamedMetadata(PrintfMetadataName())) {
      kernel_flags |= reflection::ExtKernelPropertyFlags::MayUsePrintf;
    }

    auto attributes_op_string = addSPIRVInst<kDebug>(
        spv::OpString, functionAttrStrings[F.getName()].c_str());

    // Kernel declaration
    // Ops[0] = void type
    // Ops[1] = reflection ext import
    // Ops[2] = function id
    // Ops[3] = kernel name
    // Ops[4] = number of arguments
    // Ops[5] = Flags
    // Ops[6] = Attributes
    SPIRVOperandVec Ops;
    Ops << void_id << import_id << reflection::ExtInstKernel << ValueMap[&F]
        << kernel_name << getSPIRVInt32Constant(num_args)
        << getSPIRVInt32Constant(kernel_flags) << attributes_op_string;
    auto kernel_decl = addSPIRVInst<kReflection>(spv::OpExtInst, Ops);

    // Generate the required workgroup size property if it was specified.
    if (const MDNode *MD = F.getMetadata("reqd_work_group_size")) {
      uint32_t CurXDimCst = static_cast<uint32_t>(
          mdconst::extract<ConstantInt>(MD->getOperand(0))->getZExtValue());
      uint32_t CurYDimCst = static_cast<uint32_t>(
          mdconst::extract<ConstantInt>(MD->getOperand(1))->getZExtValue());
      uint32_t CurZDimCst = static_cast<uint32_t>(
          mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue());

      Ops.clear();
      Ops << void_id << import_id
          << reflection::ExtInstPropertyRequiredWorkgroupSize << kernel_decl
          << getSPIRVInt32Constant(CurXDimCst)
          << getSPIRVInt32Constant(CurYDimCst)
          << getSPIRVInt32Constant(CurZDimCst);
      addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
    }

    auto &resource_var_at_index = FunctionToResourceVarsMap[&F];

    auto local_spec_id_md =
        module->getNamedMetadata(clspv::LocalSpecIdMetadataName());

    // Map new ordinals to old ordinals to be able to use old ordinals when
    // generating reflection for image metadata.
    std::unordered_map<uint32_t, uint32_t> ordinals_map;

    if (arg_map) {
      for (const auto &arg : arg_map->operands()) {
        const MDNode *arg_node = dyn_cast<MDNode>(arg.get());
        assert(arg_node->getNumOperands() == 6);
        const auto name =
            dyn_cast<MDString>(arg_node->getOperand(0))->getString();
        const auto old_index =
            dyn_extract<ConstantInt>(arg_node->getOperand(1))->getZExtValue();
        // Remapped argument index
        const int new_index = static_cast<int>(
            dyn_extract<ConstantInt>(arg_node->getOperand(2))->getSExtValue());
        const auto offset =
            dyn_extract<ConstantInt>(arg_node->getOperand(3))->getZExtValue();
        const auto size =
            dyn_extract<ConstantInt>(arg_node->getOperand(4))->getZExtValue();
        const auto argKind = clspv::GetArgKindFromName(
            dyn_cast<MDString>(arg_node->getOperand(5))->getString().str());

        ordinals_map[new_index] = old_index;

        // If this is a local memory argument, find the right spec id for this
        // argument.
        int64_t spec_id = -1;
        if (argKind == clspv::ArgKind::Local && local_spec_id_md) {
          for (auto spec_id_arg : local_spec_id_md->operands()) {
            if ((&F == dyn_cast<Function>(
                           dyn_cast<ValueAsMetadata>(spec_id_arg->getOperand(0))
                               ->getValue())) &&
                (static_cast<uint64_t>(new_index) ==
                 mdconst::extract<ConstantInt>(spec_id_arg->getOperand(1))
                     ->getZExtValue())) {
              spec_id =
                  mdconst::extract<ConstantInt>(spec_id_arg->getOperand(2))
                      ->getSExtValue();
              break;
            }
          }
        }

        // Generate the specific argument instruction.
        const uint32_t ordinal = static_cast<uint32_t>(old_index);
        uint32_t arg_offset = static_cast<uint32_t>(offset);
        const uint32_t arg_size = static_cast<uint32_t>(size);
        uint32_t elem_size = 0;
        uint32_t descriptor_set = 0;
        uint32_t binding = 0;
        if (spec_id > 0) {
          auto &local_arg_info = LocalSpecIdInfoMap[spec_id];
          elem_size = static_cast<uint32_t>(
              GetTypeAllocSize(local_arg_info.elem_type, DL));
        } else if (new_index >= 0) {
          if (static_cast<uint64_t>(new_index) >=
                  resource_var_at_index.size() ||
              !resource_var_at_index[new_index]) {
            // Unused
            continue;
          }
          auto *info = resource_var_at_index[new_index];
          assert(info);
          descriptor_set = info->descriptor_set;
          binding = info->binding;
        }
        AddArgumentReflection(F, kernel_decl, name.str(), argKind, ordinal,
                              descriptor_set, binding, arg_offset, arg_size,
                              static_cast<uint32_t>(spec_id), elem_size);
      }
    } else {
      // There is no argument map.
      // Take descriptor info from the resource variable calls.
      // Take argument name and size from the arguments list.

      SmallVector<Argument *, 4> arguments;
      for (auto &arg : F.args()) {
        arguments.push_back(&arg);
      }

      unsigned arg_index = 0;
      for (auto *info : resource_var_at_index) {
        if (info) {
          auto arg = arguments[arg_index];
          unsigned arg_size = 0;
          if (info->arg_kind == clspv::ArgKind::Pod ||
              info->arg_kind == clspv::ArgKind::PodUBO ||
              info->arg_kind == clspv::ArgKind::PointerUBO ||
              info->arg_kind == clspv::ArgKind::PodPushConstant) {
            arg_size =
                static_cast<uint32_t>(DL.getTypeStoreSize(arg->getType()));
          }

          // Local pointer arguments are unused in this case.
          // offset, spec_id and elem_size always 0.
          AddArgumentReflection(F, kernel_decl, arg->getName().str(),
                                info->arg_kind, arg_index, info->descriptor_set,
                                info->binding, 0, arg_size, 0, 0);
        }
        arg_index++;
      }
      // Generate mappings for pointer-to-local arguments.
      for (arg_index = 0; arg_index < arguments.size(); ++arg_index) {
        Argument *arg = arguments[arg_index];
        auto where = LocalArgSpecIds.find(arg);
        if (where != LocalArgSpecIds.end()) {
          auto &local_arg_info = LocalSpecIdInfoMap[where->second];

          // descriptor_set, binding, offset and size are always 0.
          AddArgumentReflection(F, kernel_decl, arg->getName().str(),
                                ArgKind::Local, arg_index, 0, 0, 0, 0,
                                static_cast<uint32_t>(local_arg_info.spec_id),
                                static_cast<uint32_t>(GetTypeAllocSize(
                                    local_arg_info.elem_type, DL)));
        }
      }
    }

    auto *sampler_md =
        F.getMetadata(clspv::PushConstantMetadataSamplerMaskName());
    if (sampler_md) {
      auto GV = module->getGlobalVariable(clspv::PushConstantsVariableName());
      auto STy = cast<StructType>(GV->getValueType());
      auto num_operands = sampler_md->getNumOperands();
      assert(num_operands % 2 == 0);
      for (unsigned i = 0; i < num_operands; i += 2) {
        auto ordinal =
            mdconst::extract<ConstantInt>(sampler_md->getOperand(i + 0))
                ->getZExtValue();

        // Ordinals could have changed because of pod arguments, remap it to the
        // initial ordinal if needed.
        auto find = ordinals_map.find(ordinal);
        if (find != ordinals_map.end()) {
          ordinal = find->second;
        }
        auto index =
            mdconst::extract<ConstantInt>(sampler_md->getOperand(i + 1))
                ->getZExtValue();
        auto offset = GetExplicitLayoutStructMemberOffset(
                          STy, STy->getStructNumElements() - 1, DL) +
                      GetExplicitLayoutStructMemberOffset(
                          cast<StructType>(STy->getStructElementType(
                              STy->getStructNumElements() - 1)),
                          index, DL);
        Ops.clear();
        Ops << getSPIRVType(Type::getVoidTy(module->getContext())) << import_id
            << reflection::ExtInstNormalizedSamplerMaskPushConstant
            << kernel_decl << getSPIRVInt32Constant(ordinal)
            << getSPIRVInt32Constant(offset)
            << getSPIRVInt32Constant(sizeof(uint32_t));
        addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
      }
    }

    // Generate the reflection for the image channel getter function if it is
    // used in this kernel.
    auto *image_getter_md = F.getMetadata(clspv::PushConstantsMetadataImageChannelName());
    if (image_getter_md) {
      auto GV = module->getGlobalVariable(clspv::PushConstantsVariableName());
      auto STy = cast<StructType>(GV->getValueType());
      auto num_operands = image_getter_md->getNumOperands();
      assert(num_operands % 3 == 0);
      for (unsigned i = 0; i < num_operands; i += 3) {
        auto ordinal =
            mdconst::extract<ConstantInt>(image_getter_md->getOperand(i + 0))
                ->getZExtValue();

        // Ordinals could have changed because of pod arguments, remap it to the
        // initial ordinal if needed.
        auto find = ordinals_map.find(ordinal);
        if (find != ordinals_map.end()) {
          ordinal = find->second;
        }

        auto index =
            mdconst::extract<ConstantInt>(image_getter_md->getOperand(i + 1))
                ->getZExtValue();
        auto pc = static_cast<clspv::ImageMetadata>(
            mdconst::extract<ConstantInt>(image_getter_md->getOperand(i + 2))
                ->getZExtValue());
        auto offset = GetExplicitLayoutStructMemberOffset(
                          STy, STy->getStructNumElements() - 1, DL) +
                      GetExplicitLayoutStructMemberOffset(
                          cast<StructType>(STy->getStructElementType(
                              STy->getStructNumElements() - 1)),
                          index, DL);

        reflection::ExtInst pc_inst = reflection::ExtInstMax;
        switch (pc) {
        case ImageMetadata::ChannelOrder:
          pc_inst =
              reflection::ExtInstImageArgumentInfoChannelOrderPushConstant;
          break;
        case ImageMetadata::ChannelDataType:
          pc_inst =
              reflection::ExtInstImageArgumentInfoChannelDataTypePushConstant;
          break;
        default:
          llvm_unreachable("Unhandled push constant");
          break;
        }
        Ops.clear();
        Ops << getSPIRVType(Type::getVoidTy(module->getContext())) << import_id
            << pc_inst << kernel_decl << getSPIRVInt32Constant(ordinal)
            << getSPIRVInt32Constant(offset)
            << getSPIRVInt32Constant(sizeof(uint32_t));
        addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
      }
    }
  }
}

void SPIRVProducerPassImpl::AddArgumentReflection(
    const Function &kernelFn, SPIRVID kernel_decl, const std::string &name,
    clspv::ArgKind arg_kind, uint32_t ordinal, uint32_t descriptor_set,
    uint32_t binding, uint32_t offset, uint32_t size, uint32_t spec_id,
    uint32_t elem_size) {
  // Generate ArgumentInfo for this argument.
  auto import_id = getReflectionImport();
  auto kernel_arg_name = kernelFn.getMetadata("kernel_arg_name");
  SPIRVID arg_name;
  if (kernel_arg_name) {
    arg_name = addSPIRVInst<kDebug>(
        spv::OpString, dyn_cast<MDString>(kernel_arg_name->getOperand(ordinal))
                           ->getString()
                           .str()
                           .c_str());
  } else {
    // For legacy purpose
    arg_name = addSPIRVInst<kDebug>(spv::OpString, name.c_str());
  }
  auto void_id = getSPIRVType(Type::getVoidTy(module->getContext()));
  SPIRVOperandVec Ops;
  Ops << void_id << import_id << reflection::ExtInstArgumentInfo << arg_name;

  if (clspv::Option::KernelArgInfo()) {
    assert(kernelFn.getMetadata("kernel_arg_type") &&
            kernelFn.getMetadata("kernel_arg_addr_space") &&
            kernelFn.getMetadata("kernel_arg_access_qual") &&
            kernelFn.getMetadata("kernel_arg_type_qual")
          );
    auto const &type_op =
        kernelFn.getMetadata("kernel_arg_type")->getOperand(ordinal);
    auto const &type_name_str = dyn_cast<MDString>(type_op)->getString();
    auto type_name =
        addSPIRVInst<kDebug>(spv::OpString, type_name_str.str().c_str());
    Ops << type_name;

    auto const &addrspace_op =
        kernelFn.getMetadata("kernel_arg_addr_space")->getOperand(ordinal);
    auto addrspace =
        mdconst::extract<ConstantInt>(addrspace_op)->getZExtValue();
    unsigned addrspace_enum_value = 0x119E; // CL_KERNEL_ARG_ADDRESS_PRIVATE
    switch (addrspace) {
    case clspv::AddressSpace::Global:
      addrspace_enum_value = 0x119B; // CL_KERNEL_ARG_ADDRESS_GLOBAL
      break;
    case clspv::AddressSpace::Constant:
      addrspace_enum_value = 0x119D; // CL_KERNEL_ARG_ADDRESS_CONSTANT
      break;
    case clspv::AddressSpace::Local:
      addrspace_enum_value = 0x119C; // CL_KERNEL_ARG_ADDRESS_LOCAL
      break;
    }
    auto addrspace_enum = getSPIRVInt32Constant(addrspace_enum_value);
    Ops << addrspace_enum;

    auto const &access_qual_op =
        kernelFn.getMetadata("kernel_arg_access_qual")->getOperand(ordinal);
    auto const &access_qual_str =
        dyn_cast<MDString>(access_qual_op)->getString();
    unsigned access_qual_enum_value =
        StringSwitch<unsigned>(access_qual_str)
            .Case("read_only", 0x11A0)  // CL_KERNEL_ARG_ACCESS_READ_ONLY
            .Case("write_only", 0x11A1) // CL_KERNEL_ARG_ACCESS_WRITE_ONLY
            .Case("read_write", 0x11A2) // CL_KERNEL_ARG_ACCESS_READ_WRITE
            .Default(0x11A3);           // CL_KERNEL_ARG_ACCESS_NONE
    auto access_qual_enum = getSPIRVInt32Constant(access_qual_enum_value);
    Ops << access_qual_enum;

    auto const &type_qual_op =
        kernelFn.getMetadata("kernel_arg_type_qual")->getOperand(ordinal);
    auto const &type_qual_str = dyn_cast<MDString>(type_qual_op)->getString();
    unsigned type_qual_enum_value = 0; // CL_KERNEL_ARG_TYPE_NONE
    if (type_qual_str.find("const") != std::string::npos) {
      type_qual_enum_value |= (1 << 0); // CL_KERNEL_ARG_TYPE_CONST
    }
    if (type_qual_str.find("restrict") != std::string::npos) {
      type_qual_enum_value |= (1 << 1); // CL_KERNEL_ARG_TYPE_RESTRICT
    }
    if (type_qual_str.find("volatile") != std::string::npos) {
      type_qual_enum_value |= (1 << 2); // CL_KERNEL_ARG_TYPE_VOLATILE
    }
    if (type_qual_str.find("pipe") != std::string::npos) {
      type_qual_enum_value |= (1 << 3); // CL_KERNEL_ARG_TYPE_PIPE
    }
    auto type_qual_enum = getSPIRVInt32Constant(type_qual_enum_value);
    Ops << type_qual_enum;
  
  }

  auto arg_info = addSPIRVInst<kReflection>(spv::OpExtInst, Ops);

  Ops.clear();
  Ops << void_id << import_id;
  reflection::ExtInst ext_inst = reflection::ExtInstMax;
  // Determine the extended instruction.
  switch (arg_kind) {
  case clspv::ArgKind::Buffer:
    ext_inst = reflection::ExtInstArgumentStorageBuffer;
    break;
  case clspv::ArgKind::BufferUBO:
    ext_inst = reflection::ExtInstArgumentUniform;
    break;
  case clspv::ArgKind::Local:
    ext_inst = reflection::ExtInstArgumentWorkgroup;
    break;
  case clspv::ArgKind::Pod:
    ext_inst = reflection::ExtInstArgumentPodStorageBuffer;
    break;
  case clspv::ArgKind::PodUBO:
    ext_inst = reflection::ExtInstArgumentPodUniform;
    break;
  case clspv::ArgKind::PodPushConstant:
    ext_inst = reflection::ExtInstArgumentPodPushConstant;
    break;
  case clspv::ArgKind::PointerPushConstant:
    ext_inst = reflection::ExtInstArgumentPointerPushConstant;
    break;
  case clspv::ArgKind::PointerUBO:
    ext_inst = reflection::ExtInstArgumentPointerUniform;
    break;
  case clspv::ArgKind::SampledImage:
    ext_inst = reflection::ExtInstArgumentSampledImage;
    break;
  case clspv::ArgKind::StorageImage:
    ext_inst = reflection::ExtInstArgumentStorageImage;
    break;
  case clspv::ArgKind::Sampler:
    ext_inst = reflection::ExtInstArgumentSampler;
    break;
  case clspv::ArgKind::StorageTexelBuffer:
    ext_inst = reflection::ExtInstArgumentStorageTexelBuffer;
    break;
  case clspv::ArgKind::UniformTexelBuffer:
    ext_inst = reflection::ExtInstArgumentUniformTexelBuffer;
    break;
  default:
    llvm_unreachable("Unhandled argument reflection");
    break;
  }
  Ops << ext_inst << kernel_decl << getSPIRVInt32Constant(ordinal);

  // Add descriptor set and binding for applicable arguments.
  switch (arg_kind) {
  case clspv::ArgKind::Buffer:
  case clspv::ArgKind::BufferUBO:
  case clspv::ArgKind::Pod:
  case clspv::ArgKind::PodUBO:
  case clspv::ArgKind::PointerUBO:
  case clspv::ArgKind::SampledImage:
  case clspv::ArgKind::StorageImage:
  case clspv::ArgKind::Sampler:
  case clspv::ArgKind::StorageTexelBuffer:
  case clspv::ArgKind::UniformTexelBuffer:
    Ops << getSPIRVInt32Constant(descriptor_set)
        << getSPIRVInt32Constant(binding);
    break;
  default:
    break;
  }

  // Add remaining operands for arguments.
  switch (arg_kind) {
  case clspv::ArgKind::Local:
    Ops << getSPIRVInt32Constant(spec_id) << getSPIRVInt32Constant(elem_size);
    break;
  case clspv::ArgKind::Pod:
  case clspv::ArgKind::PodUBO:
  case clspv::ArgKind::PodPushConstant:
  case clspv::ArgKind::PointerPushConstant:
  case clspv::ArgKind::PointerUBO:
    Ops << getSPIRVInt32Constant(offset) << getSPIRVInt32Constant(size);
    break;
  default:
    break;
  }
  Ops << arg_info;
  addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
}

void SPIRVProducerPassImpl::GeneratePrintfReflection() {
  auto import_id = getReflectionImport();
  auto void_id = getSPIRVType(Type::getVoidTy(module->getContext()));

  auto *PrintfMD = module->getNamedMetadata(PrintfMetadataName());
  if (!PrintfMD) {
    return;
  }

  SPIRVOperandVec Ops;

  for (auto *PrintMD : PrintfMD->operands()) {
    Ops << void_id << import_id << reflection::ExtInstPrintfInfo;

    assert(PrintMD->getNumOperands() == 3);
    auto *PrintfID = dyn_cast<ConstantAsMetadata>(PrintMD->getOperand(0).get());
    auto *PrintfString = dyn_cast<MDString>(PrintMD->getOperand(1).get());
    auto *PrintfArgs = dyn_cast<MDTuple>(PrintMD->getOperand(2).get());
    auto PrintfStringConstant = addSPIRVInst<kDebug>(
        spv::OpString, PrintfString->getString().str().c_str());

    Ops << getSPIRVInt32Constant(static_cast<int32_t>(
        mdconst::extract<ConstantInt>(PrintfID)->getZExtValue()));
    Ops << PrintfStringConstant;
    for (auto &ArgSizeOperand : PrintfArgs->operands()) {
      auto *ArgSizeConst = dyn_cast<ConstantAsMetadata>(ArgSizeOperand.get());
      int32_t ArgSizeVal = static_cast<int32_t>(
          mdconst::extract<ConstantInt>(ArgSizeConst)->getZExtValue());
      Ops << getSPIRVInt32Constant(ArgSizeVal);
    }
    addSPIRVInst<kReflection>(spv::OpExtInst, Ops);
  }
}