https://github.com/farzonl created 
https://github.com/llvm/llvm-project/pull/185471

fixes #184877

This change was threefold.
1. copy the padded cbuffer from memory to a local alloca
2. switch to using the new `getFlattenedIndex` helpers for index generation
3. convert row major to column major indicies in codegen depending on 
LangOptions

>From f4864036f96f2e23935e30ae6805c300ba90ef53 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <[email protected]>
Date: Mon, 9 Mar 2026 13:18:43 -0400
Subject: [PATCH] [Matrix] Fix cbuffers support for matrix element expr

fixes #184877

This change was threefold.
1. copy the padded cbuffer from memory to a local alloca
2. switch to using the new `getFlattenedIndex` helpers for index
   generation
3. convert row major to column major indicies in codegen depending on 
LangOptions
---
 clang/lib/AST/Expr.cpp                        | 24 ++---
 clang/lib/CodeGen/CGExpr.cpp                  | 31 +++++-
 .../MatrixElementRowColFlags.hlsl             | 37 +++++++
 ...member-one-based-accessor-scalar-load.hlsl | 24 ++---
 ...ember-one-based-accessor-scalar-store.hlsl | 24 ++---
 .../matrix-member-one-based-swizzle-load.hlsl | 16 ++--
 ...matrix-member-one-based-swizzle-store.hlsl | 48 +++++-----
 ...ember-zero-based-accessor-scalar-load.hlsl | 24 ++---
 ...mber-zero-based-accessor-scalar-store.hlsl | 24 ++---
 ...matrix-member-zero-based-swizzle-load.hlsl | 16 ++--
 ...atrix-member-zero-based-swizzle-store.hlsl | 48 +++++-----
 .../resources/MatrixElement_cbuffer.hlsl      | 96 +++++++++++++++++++
 12 files changed, 285 insertions(+), 127 deletions(-)
 create mode 100644 
clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
 create mode 100644 clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl

diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 9632d88fae4e4..185e887fb05c3 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4478,8 +4478,8 @@ static MatrixAccessorFormat 
GetHLSLMatrixAccessorFormat(StringRef Comp) {
 }
 
 template <typename Fn>
-static bool ForEachMatrixAccessorIndex(StringRef Comp, unsigned Rows,
-                                       unsigned Cols, Fn &&F) {
+static bool ForEachMatrixAccessorIndex(StringRef Comp,
+                                       const ConstantMatrixType *MT, Fn &&F) {
   auto Format = GetHLSLMatrixAccessorFormat(Comp);
 
   for (unsigned I = 0, E = Comp.size(); I < E; I += Format.ChunkLen) {
@@ -4491,8 +4491,13 @@ static bool ForEachMatrixAccessorIndex(StringRef Comp, 
unsigned Rows,
     Col = static_cast<unsigned>(Comp[I + ZeroIndexOffset + 2] - '0') -
           OneIndexOffset;
 
-    assert(Row < Rows && Col < Cols && "matrix swizzle index out of bounds");
-    const unsigned Index = Row * Cols + Col;
+    assert(Row < MT->getNumRows() && Col < MT->getNumColumns() &&
+           "matrix swizzle index out of bounds");
+    // NOTE: AST layer has no access to LangOptions so we will default to row
+    // major b\c all other AST matrix representations are row major.
+    // However in codegen we need to convert to column major if the flag
+    // requires it.
+    const unsigned Index = MT->getFlattenedIndex(Row, Col, /*IsRowMajor*/ 
true);
     // Callback returns true to continue, false to stop early.
     if (!F(Index))
       return false;
@@ -4507,13 +4512,10 @@ static bool ForEachMatrixAccessorIndex(StringRef Comp, 
unsigned Rows,
 bool MatrixElementExpr::containsDuplicateElements() const {
   StringRef Comp = Accessor->getName();
   const auto *MT = getBase()->getType()->castAs<ConstantMatrixType>();
-  const unsigned Rows = MT->getNumRows();
-  const unsigned Cols = MT->getNumColumns();
-  const unsigned Max = Rows * Cols;
 
-  llvm::BitVector Seen(Max, /*t=*/false);
+  llvm::BitVector Seen(MT->getNumElementsFlattened(), /*t=*/false);
   bool HasDup = false;
-  ForEachMatrixAccessorIndex(Comp, Rows, Cols, [&](unsigned Index) -> bool {
+  ForEachMatrixAccessorIndex(Comp, MT, [&](unsigned Index) -> bool {
     if (Seen[Index]) {
       HasDup = true;
       return false; // exit early
@@ -4562,9 +4564,7 @@ void MatrixElementExpr::getEncodedElementAccess(
     SmallVectorImpl<uint32_t> &Elts) const {
   StringRef Comp = Accessor->getName();
   const auto *MT = getBase()->getType()->castAs<ConstantMatrixType>();
-  const unsigned Rows = MT->getNumRows();
-  const unsigned Cols = MT->getNumColumns();
-  ForEachMatrixAccessorIndex(Comp, Rows, Cols, [&](unsigned Index) -> bool {
+  ForEachMatrixAccessorIndex(Comp, MT, [&](unsigned Index) -> bool {
     Elts.push_back(Index);
     return true;
   });
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index eebb36276e0eb..fe5de9f8df09d 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2329,15 +2329,39 @@ LValue CodeGenFunction::EmitMatrixElementExpr(const 
MatrixElementExpr *E) {
       E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
 
   // Encode the element access list into a vector of unsigned indices.
+  // getEncodedElementAccess returns row-major linearized indices.
   SmallVector<uint32_t, 4> Indices;
   E->getEncodedElementAccess(Indices);
 
+  // getEncodedElementAccess returns row-major linearized indices
+  // If the matrix memory layout is column-major, convert indices
+  // to column-major indices.
+  bool IsColMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
+                    LangOptions::MatrixMemoryLayout::MatrixColMajor;
+  if (IsColMajor) {
+    const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
+    unsigned NumCols = MT->getNumColumns();
+    for (uint32_t &Idx : Indices) {
+      // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
+      unsigned Row = Idx / NumCols;
+      unsigned Col = Idx % NumCols;
+      // Re-linearize as column-major
+      Idx = MT->getColumnMajorFlattenedIndex(Row, Col);
+    }
+  }
+
   if (Base.isSimple()) {
+    RawAddress MatAddr = Base.getAddress();
+    if (getLangOpts().HLSL &&
+        E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
+      MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(
+          Base, E->getExprLoc(), *this);
+
     llvm::Constant *CV =
         llvm::ConstantDataVector::get(getLLVMContext(), Indices);
-    return LValue::MakeExtVectorElt(
-        MaybeConvertMatrixAddress(Base.getAddress(), *this), CV, ResultType,
-        Base.getBaseInfo(), TBAAAccessInfo());
+    return LValue::MakeExtVectorElt(MaybeConvertMatrixAddress(MatAddr, *this),
+                                    CV, ResultType, Base.getBaseInfo(),
+                                    TBAAAccessInfo());
   }
   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
 
@@ -2347,6 +2371,7 @@ LValue CodeGenFunction::EmitMatrixElementExpr(const 
MatrixElementExpr *E) {
   for (unsigned Index : Indices)
     CElts.push_back(BaseElts->getAggregateElement(Index));
   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
+
   return LValue::MakeExtVectorElt(
       MaybeConvertMatrixAddress(Base.getExtVectorAddress(), *this), CV,
       ResultType, Base.getBaseInfo(), TBAAAccessInfo());
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl 
b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
new file mode 100644
index 0000000000000..fe9e69c6a6898
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes 
\
+// RUN:   -emit-llvm -finclude-default-header 
-fmatrix-memory-layout=column-major \
+// RUN:   -o - %s | FileCheck %s --check-prefixes=CHECK,COL
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes 
\
+// RUN:   -emit-llvm -finclude-default-header -fmatrix-memory-layout=row-major 
\
+// RUN:   -o - %s | FileCheck %s --check-prefixes=CHECK,ROW
+
+// For a float3x2 matrix (3 rows, 2 columns):
+//   Column-major flat vector: [_11, _21, _31, _12, _22, _32]
+//                         idx:  0    1    2    3    4    5
+//   Row-major flat vector:    [_11, _12, _21, _22, _31, _32]
+//                         idx:  0    1    2    3    4    5
+
+
+// CHECK-LABEL: define {{.*}} @_Z16getScalarElementu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: extractelement <6 x float> {{.*}}, i32 4
+// ROW-NEXT: extractelement <6 x float> {{.*}}, i32 3
+export float getScalarElement(float3x2 M) {
+  return M._22;
+}
+
+// CHECK-LABEL: define {{.*}} @_Z18getSwizzleElementsu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <4 x i32> 
<i32 0, i32 3, i32 1, i32 4>
+// ROW-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <4 x i32> 
<i32 0, i32 1, i32 2, i32 3>
+export float4 getSwizzleElements(float3x2 M) {
+  return M._11_12_21_22;
+}
+
+// CHECK-LABEL: define {{.*}} 
@_Z22getZeroBasedSwizzleEltu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <2 x i32> 
<i32 1, i32 3>
+// ROW-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <2 x i32> 
<i32 2, i32 1>
+export float2 getZeroBasedSwizzleElt(float3x2 M) {
+  return M._m10_m01;
+}
\ No newline at end of file
diff --git 
a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
index def8aa5440568..6d8a3ce6ecbb6 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
@@ -23,7 +23,7 @@ int Return11(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 1
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 4
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return12(int4x4 A) {
@@ -36,7 +36,7 @@ int Return12(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 2
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 8
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return13(int4x4 A) {
@@ -49,7 +49,7 @@ int Return13(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 3
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 12
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return14(int4x4 A) {
@@ -62,7 +62,7 @@ int Return14(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 4
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 1
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return21(int4x4 A) {
@@ -88,7 +88,7 @@ int Return22(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 6
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 9
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return23(int4x4 A) {
@@ -101,7 +101,7 @@ int Return23(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 7
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 13
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return24(int4x4 A) {
@@ -114,7 +114,7 @@ int Return24(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 8
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 2
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return31(int4x4 A) {
@@ -127,7 +127,7 @@ int Return31(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 9
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 6
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return32(int4x4 A) {
@@ -153,7 +153,7 @@ int Return33(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 11
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 14
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return34(int4x4 A) {
@@ -166,7 +166,7 @@ int Return34(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 12
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 3
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return41(int4x4 A) {
@@ -179,7 +179,7 @@ int Return41(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 13
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 7
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return42(int4x4 A) {
@@ -192,7 +192,7 @@ int Return42(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 14
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 11
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
 int Return43(int4x4 A) {
diff --git 
a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
index fb3a46170ebe0..9ea292ecea007 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
@@ -29,7 +29,7 @@ void StoreScalarAtMat11(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 1
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 4
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -46,7 +46,7 @@ void StoreScalarAtMat12(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 2
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 8
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -63,7 +63,7 @@ void StoreScalarAtMat13(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 3
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 12
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -80,7 +80,7 @@ void StoreScalarAtMat14(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 1
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -114,7 +114,7 @@ void StoreScalarAtMat22(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 6
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 9
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -131,7 +131,7 @@ void StoreScalarAtMat23(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 7
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 13
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -148,7 +148,7 @@ void StoreScalarAtMat24(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 8
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 2
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -165,7 +165,7 @@ void StoreScalarAtMat31(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 9
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 6
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -199,7 +199,7 @@ void StoreScalarAtMat33(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 11
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 14
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -216,7 +216,7 @@ void StoreScalarAtMat34(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 12
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 3
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -233,7 +233,7 @@ void StoreScalarAtMat41(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 13
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 7
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -250,7 +250,7 @@ void StoreScalarAtMat42(out int4x4 A, int I) {
 // CHECK-NEXT:    store i32 [[I]], ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 14
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 11
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
index 47737aaab0390..6567d49f04dcf 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
@@ -9,7 +9,7 @@
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnOnesSwizzle(int4x4 A) {
@@ -22,7 +22,7 @@ int4 ReturnOnesSwizzle(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnOnesSwizzle2(int4x4 A) {
@@ -35,7 +35,7 @@ int4 ReturnOnesSwizzle2(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnTwosSwizzle(int4x4 A) {
@@ -48,7 +48,7 @@ int4 ReturnTwosSwizzle(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnTwosSwizzle2(int4x4 A) {
@@ -61,7 +61,7 @@ int4 ReturnTwosSwizzle2(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnThreesSwizzle(int4x4 A) {
@@ -74,7 +74,7 @@ int4 ReturnThreesSwizzle(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnThreesSwizzle2(int4x4 A) {
@@ -87,7 +87,7 @@ int4 ReturnThreesSwizzle2(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnFoursSwizzle(int4x4 A) {
@@ -100,7 +100,7 @@ int4 ReturnFoursSwizzle(int4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
 // CHECK-NEXT:    store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> 
poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 //
 int4 ReturnFoursSwizzle2(int4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
index ff7fab662a012..26b16d02a0d4a 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
@@ -17,13 +17,13 @@
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 4
 // CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP5]], align 4
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 2
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 8
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[TMP7]], align 4
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 3
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 12
 // CHECK-NEXT:    store i32 [[TMP8]], ptr [[TMP9]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -43,13 +43,13 @@ void OnesSwizzleToScalar(out int4x4 A, int I) {
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
 // CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP1]], align 4
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 1
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 8
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 2
 // CHECK-NEXT:    store i32 [[TMP5]], ptr [[TMP6]], align 4
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 12
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 3
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -69,16 +69,16 @@ void OnesSwizzleToVector(out int4x4 A, int4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x 
i32> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 1
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
 // CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 5
 // CHECK-NEXT:    store i32 [[TMP5]], ptr [[TMP6]], align 4
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 6
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 9
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 7
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 13
 // CHECK-NEXT:    store i32 [[TMP9]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -96,16 +96,16 @@ void TwosSwizzleToScalar(out int4x4 A, int I) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 1
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 4
 // CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 5
 // CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP5]], align 4
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 9
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 6
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[TMP7]], align 4
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 13
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 7
 // CHECK-NEXT:    store i32 [[TMP8]], ptr [[TMP9]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -125,16 +125,16 @@ void TwosSwizzleToVector(out int4x4 A, int4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x 
i32> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 8
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 2
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 9
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 6
 // CHECK-NEXT:    store i32 [[TMP5]], ptr [[TMP6]], align 4
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
 // CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 10
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 11
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 14
 // CHECK-NEXT:    store i32 [[TMP9]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -152,16 +152,16 @@ void ThreesSwizzleToScalar(out int4x4 A, int I) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 2
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 8
 // CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 6
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 9
 // CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP5]], align 4
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
 // CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 10
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[TMP7]], align 4
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 14
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 11
 // CHECK-NEXT:    store i32 [[TMP8]], ptr [[TMP9]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -181,13 +181,13 @@ void ThreesSwizzleToVector(out int4x4 A, int4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x 
i32> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 12
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 3
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 13
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 7
 // CHECK-NEXT:    store i32 [[TMP5]], ptr [[TMP6]], align 4
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 14
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 11
 // CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
 // CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 
0, i32 15
@@ -208,13 +208,13 @@ void FoursSwizzleToScalar(out int4x4 A, int I) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 12
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 3
 // CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 13
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 7
 // CHECK-NEXT:    store i32 [[TMP4]], ptr [[TMP5]], align 4
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 14
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 11
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[TMP7]], align 4
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
 // CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 
0, i32 15
diff --git 
a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
index 5d542b8647eac..def0b884290da 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
@@ -23,7 +23,7 @@ float Return00(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 1
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 4
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return01(float4x4 A) {
@@ -36,7 +36,7 @@ float Return01(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 2
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 8
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return02(float4x4 A) {
@@ -49,7 +49,7 @@ float Return02(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 3
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 12
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return03(float4x4 A) {
@@ -62,7 +62,7 @@ float Return03(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 4
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 1
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return10(float4x4 A) {
@@ -88,7 +88,7 @@ float Return11(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 6
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 9
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return12(float4x4 A) {
@@ -101,7 +101,7 @@ float Return12(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 7
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 13
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return13(float4x4 A) {
@@ -114,7 +114,7 @@ float Return13(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 8
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 2
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return20(float4x4 A) {
@@ -127,7 +127,7 @@ float Return20(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 9
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 6
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return21(float4x4 A) {
@@ -153,7 +153,7 @@ float Return22(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 11
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 14
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return23(float4x4 A) {
@@ -166,7 +166,7 @@ float Return23(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 12
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 3
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return30(float4x4 A) {
@@ -179,7 +179,7 @@ float Return30(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 13
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 7
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return31(float4x4 A) {
@@ -192,7 +192,7 @@ float Return31(float4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
 // CHECK-NEXT:    store <16 x float> [[A]], ptr [[A_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 14
+// CHECK-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 11
 // CHECK-NEXT:    ret float [[TMP1]]
 //
 float Return32(float4x4 A) {
diff --git 
a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
index 97dc7e7c2bd37..ab5b69cb794b6 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
@@ -29,7 +29,7 @@ void StoreScalarAtMat00(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 1
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 4
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -46,7 +46,7 @@ void StoreScalarAtMat01(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 2
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 8
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -63,7 +63,7 @@ void StoreScalarAtMat02(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 3
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 12
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -80,7 +80,7 @@ void StoreScalarAtMat03(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 4
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 1
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -114,7 +114,7 @@ void StoreScalarAtMat11(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 6
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 9
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -131,7 +131,7 @@ void StoreScalarAtMat12(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 7
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 13
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -148,7 +148,7 @@ void StoreScalarAtMat13(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 8
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 2
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -165,7 +165,7 @@ void StoreScalarAtMat20(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 9
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 6
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -199,7 +199,7 @@ void StoreScalarAtMat22(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 11
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 14
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -216,7 +216,7 @@ void StoreScalarAtMat23(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 12
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 3
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -233,7 +233,7 @@ void StoreScalarAtMat30(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 13
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 7
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -250,7 +250,7 @@ void StoreScalarAtMat31(out float4x4 A, float F) {
 // CHECK-NEXT:    store float [[F]], ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 14
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 
0, i32 11
 // CHECK-NEXT:    store float [[TMP0]], ptr [[TMP2]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
index dca2e6132de60..d9daf6ed7305b 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
@@ -9,7 +9,7 @@
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnZerosSwizzle(half4x4 A) {
@@ -22,7 +22,7 @@ half4 ReturnZerosSwizzle(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnZerosSwizzle2(half4x4 A) {
@@ -35,7 +35,7 @@ half4 ReturnZerosSwizzle2(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnOnesSwizzle(half4x4 A) {
@@ -48,7 +48,7 @@ half4 ReturnOnesSwizzle(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnOnesSwizzle2(half4x4 A) {
@@ -61,7 +61,7 @@ half4 ReturnOnesSwizzle2(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnTwosSwizzle(half4x4 A) {
@@ -74,7 +74,7 @@ half4 ReturnTwosSwizzle(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnTwosSwizzle2(half4x4 A) {
@@ -87,7 +87,7 @@ half4 ReturnTwosSwizzle2(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnThreesSwizzle(half4x4 A) {
@@ -100,7 +100,7 @@ half4 ReturnThreesSwizzle(half4x4 A) {
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
 // CHECK-NEXT:    store <16 x half> [[A]], ptr [[A_ADDR]], align 2
 // CHECK-NEXT:    [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x 
half> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 // CHECK-NEXT:    ret <4 x half> [[TMP1]]
 //
 half4 ReturnThreesSwizzle2(half4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl 
b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
index 72fbc015d5934..986cdf136631b 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
@@ -17,13 +17,13 @@
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
 // CHECK-NEXT:    store double [[TMP3]], ptr [[TMP2]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 4
 // CHECK-NEXT:    store double [[TMP4]], ptr [[TMP5]], align 8
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 2
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 8
 // CHECK-NEXT:    store double [[TMP6]], ptr [[TMP7]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 3
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 12
 // CHECK-NEXT:    store double [[TMP8]], ptr [[TMP9]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -43,13 +43,13 @@ void ZerosSwizzleToScalar(out double4x4 A, double D) {
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
 // CHECK-NEXT:    store double [[TMP2]], ptr [[TMP1]], align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 1
 // CHECK-NEXT:    store double [[TMP3]], ptr [[TMP4]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 8
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 2
 // CHECK-NEXT:    store double [[TMP5]], ptr [[TMP6]], align 8
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 12
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 3
 // CHECK-NEXT:    store double [[TMP7]], ptr [[TMP8]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -69,16 +69,16 @@ void ZerosSwizzleToVector(out double4x4 A, double4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 
x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 4
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 1
 // CHECK-NEXT:    store double [[TMP3]], ptr [[TMP4]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
 // CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 5
 // CHECK-NEXT:    store double [[TMP5]], ptr [[TMP6]], align 8
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 6
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 9
 // CHECK-NEXT:    store double [[TMP7]], ptr [[TMP8]], align 8
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 7
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 13
 // CHECK-NEXT:    store double [[TMP9]], ptr [[TMP10]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -96,16 +96,16 @@ void OnesSwizzleToScalar(out double4x4 A, double D) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 1
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 4
 // CHECK-NEXT:    store double [[TMP2]], ptr [[TMP3]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 5
 // CHECK-NEXT:    store double [[TMP4]], ptr [[TMP5]], align 8
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 9
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 6
 // CHECK-NEXT:    store double [[TMP6]], ptr [[TMP7]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 13
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 7
 // CHECK-NEXT:    store double [[TMP8]], ptr [[TMP9]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -125,16 +125,16 @@ void OnesSwizzleToVector(out double4x4 A, double4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 
x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 8
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 2
 // CHECK-NEXT:    store double [[TMP3]], ptr [[TMP4]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 9
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 6
 // CHECK-NEXT:    store double [[TMP5]], ptr [[TMP6]], align 8
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
 // CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 10
 // CHECK-NEXT:    store double [[TMP7]], ptr [[TMP8]], align 8
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 11
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 14
 // CHECK-NEXT:    store double [[TMP9]], ptr [[TMP10]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -152,16 +152,16 @@ void TwosSwizzleToScalar(out double4x4 A, double D) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 2
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 8
 // CHECK-NEXT:    store double [[TMP2]], ptr [[TMP3]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 6
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 9
 // CHECK-NEXT:    store double [[TMP4]], ptr [[TMP5]], align 8
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
 // CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 10
 // CHECK-NEXT:    store double [[TMP6]], ptr [[TMP7]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 14
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 11
 // CHECK-NEXT:    store double [[TMP8]], ptr [[TMP9]], align 8
 // CHECK-NEXT:    ret void
 //
@@ -181,13 +181,13 @@ void TwosSwizzleToVector(out double4x4 A, double4 V) {
 // CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 
x double> poison, <4 x i32> zeroinitializer
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 12
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 3
 // CHECK-NEXT:    store double [[TMP3]], ptr [[TMP4]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 13
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 7
 // CHECK-NEXT:    store double [[TMP5]], ptr [[TMP6]], align 8
 // CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 14
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 11
 // CHECK-NEXT:    store double [[TMP7]], ptr [[TMP8]], align 8
 // CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
 // CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], 
i32 0, i32 15
@@ -208,13 +208,13 @@ void ThreesSwizzleToScalar(out double4x4 A, double D) {
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull 
[[META4]], !align [[META5]]
 // CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 3
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 12
 // CHECK-NEXT:    store double [[TMP2]], ptr [[TMP3]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 7
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 13
 // CHECK-NEXT:    store double [[TMP4]], ptr [[TMP5]], align 8
 // CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 11
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 14
 // CHECK-NEXT:    store double [[TMP6]], ptr [[TMP7]], align 8
 // CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
 // CHECK-NEXT:    [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], 
i32 0, i32 15
diff --git a/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl 
b/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl
new file mode 100644
index 0000000000000..e0ef9ce92fa81
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl
@@ -0,0 +1,96 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes 
\
+// RUN:   -emit-llvm -finclude-default-header 
-fmatrix-memory-layout=column-major \
+// RUN:   -o - %s | FileCheck %s --check-prefix=CHECK,COL-CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes 
\
+// RUN:   -emit-llvm -finclude-default-header -fmatrix-memory-layout=row-major 
\
+// RUN:   -o - %s | FileCheck %s --check-prefixes=CHECK,ROW-CHECK
+
+
+cbuffer CB {
+  float3x2 Mat;
+};
+
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float 
@_Z23getCBufferScalarElementv(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// COL-CHECK-NEXT:    [[MATRIX_BUF_COPY:%.*]] = alloca [2 x <3 x float>], 
align 4
+// COL-CHECK-NEXT:    [[CBUF_DEST:%.*]] = getelementptr inbounds [2 x <3 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// COL-CHECK-NEXT:    [[CBUF_LOAD:%.*]] = load <3 x float>, ptr addrspace(2) 
@Mat, align 4
+// COL-CHECK-NEXT:    store <3 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], 
align 4
+// COL-CHECK-NEXT:    [[CBUF_DEST1:%.*]] = getelementptr inbounds [2 x <3 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// COL-CHECK-NEXT:    [[CBUF_LOAD2:%.*]] = load <3 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 4
+// COL-CHECK-NEXT:    store <3 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], 
align 4
+// COL-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr 
[[MATRIX_BUF_COPY]], align 4
+// COL-CHECK-NEXT:    [[TMP1:%.*]] = extractelement <6 x float> [[TMP0]], i32 4
+// COL-CHECK-NEXT:    ret float [[TMP1]]
+//
+// ROW-CHECK-NEXT:    [[MATRIX_BUF_COPY:%.*]] = alloca [3 x <2 x float>], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// ROW-CHECK-NEXT:    [[CBUF_LOAD:%.*]] = load <2 x float>, ptr addrspace(2) 
@Mat, align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST1:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// ROW-CHECK-NEXT:    [[CBUF_LOAD2:%.*]] = load <2 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST3:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 2
+// ROW-CHECK-NEXT:    [[CBUF_LOAD4:%.*]] = load <2 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 32), align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD4]], ptr [[CBUF_DEST3]], 
align 8
+// ROW-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr 
[[MATRIX_BUF_COPY]], align 8
+// ROW-CHECK-NEXT:    [[TMP1:%.*]] = extractelement <6 x float> [[TMP0]], i32 3
+// ROW-CHECK-NEXT:    ret float [[TMP1]]
+//
+float getCBufferScalarElement() {
+  return Mat._22;
+}
+
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <4 x float> 
@_Z23getCBufferSwizzleAccessv(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// COL-CHECK-NEXT:    [[MATRIX_BUF_COPY:%.*]] = alloca [2 x <3 x float>], 
align 4
+// COL-CHECK-NEXT:    [[CBUF_DEST:%.*]] = getelementptr inbounds [2 x <3 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// COL-CHECK-NEXT:    [[CBUF_LOAD:%.*]] = load <3 x float>, ptr addrspace(2) 
@Mat, align 4
+// COL-CHECK-NEXT:    store <3 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], 
align 4
+// COL-CHECK-NEXT:    [[CBUF_DEST1:%.*]] = getelementptr inbounds [2 x <3 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// COL-CHECK-NEXT:    [[CBUF_LOAD2:%.*]] = load <3 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 4
+// COL-CHECK-NEXT:    store <3 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], 
align 4
+// COL-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr 
[[MATRIX_BUF_COPY]], align 4
+// COL-CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x 
float> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 4>
+// COL-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// ROW-CHECK-NEXT:    [[MATRIX_BUF_COPY:%.*]] = alloca [3 x <2 x float>], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// ROW-CHECK-NEXT:    [[CBUF_LOAD:%.*]] = load <2 x float>, ptr addrspace(2) 
@Mat, align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST1:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// ROW-CHECK-NEXT:    [[CBUF_LOAD2:%.*]] = load <2 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], 
align 8
+// ROW-CHECK-NEXT:    [[CBUF_DEST3:%.*]] = getelementptr inbounds [3 x <2 x 
float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 2
+// ROW-CHECK-NEXT:    [[CBUF_LOAD4:%.*]] = load <2 x float>, ptr addrspace(2) 
getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 32), align 8
+// ROW-CHECK-NEXT:    store <2 x float> [[CBUF_LOAD4]], ptr [[CBUF_DEST3]], 
align 8
+// ROW-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr 
[[MATRIX_BUF_COPY]], align 8
+// ROW-CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x 
float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// ROW-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float4 getCBufferSwizzleAccess() {
+  return Mat._11_12_21_22;
+}
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <2 x float> 
@_Z22getZeroBasedSwizzleEltu11matrix_typeILm3ELm2EfE(
+// CHECK-SAME: <6 x float> noundef nofpclass(nan inf) [[M:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// COL-CHECK-NEXT:    [[M_ADDR:%.*]] = alloca [2 x <3 x float>], align 4
+// COL-CHECK-NEXT:    store <6 x float> [[M]], ptr [[M_ADDR]], align 4
+// COL-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr [[M_ADDR]], align 4
+// COL-CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x 
float> poison, <2 x i32> <i32 1, i32 3>
+// COL-CHECK-NEXT:    ret <2 x float> [[TMP1]]
+//
+// ROW-CHECK-NEXT:    [[M_ADDR:%.*]] = alloca [3 x <2 x float>], align 4
+// ROW-CHECK-NEXT:    store <6 x float> [[M]], ptr [[M_ADDR]], align 4
+// ROW-CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr [[M_ADDR]], align 4
+// ROW-CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x 
float> poison, <2 x i32> <i32 2, i32 1>
+// ROW-CHECK-NEXT:    ret <2 x float> [[TMP1]]
+//
+float2 getZeroBasedSwizzleElt(float3x2 M) {
+  return M._m10_m01;
+}

_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Reply via email to