Support ESSL structs containing samplers on D3D

Since HLSL can't natively handle samplers in structs, samplers need to
be extracted out of structs into separate variables in the translated
shader code. In HLSL 4.1, samplers that were in structs go into the
normal sampler arrays and are identified by index constants. In other
HLSL versions, samplers that were in structs are translated as uniform
variables.

These transformations are done inside the HLSL output classes, not as
tree transformations. This helps to keep the uniform API provided by
the shader translator intact.

Wherever a struct containing samplers is passed into a user-defined
function, the translated HLSL code passes the separate sampler
variables alongside a struct where the samplers have been removed.

The D3D backend in libANGLE queries the uniform registers of any
samplers that were in uniform structs, and adds them to the register
maps, so that correct sampler state gets assigned to them.

The extracted sampler variables are prefixed with "angle_" instead of
the usual "_" to prevent any name conflicts between them and regular
variables.

BUG=angleproject:504
TEST=angle_end2end_tests,
     dEQP-GLES*.functional.shaders.struct.uniform.* (all pass),
     dEQP-GLES*.functional.uniform_api.* (most now pass)

Change-Id: Ib79cba2fa0ff8257a973d70dfd917a64f0ca1efb
Reviewed-on: https://chromium-review.googlesource.com/333743
Reviewed-by: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Olli Etuaho <oetuaho@nvidia.com>
diff --git a/src/compiler/translator/OutputHLSL.cpp b/src/compiler/translator/OutputHLSL.cpp
index b148dba..8b914b1 100644
--- a/src/compiler/translator/OutputHLSL.cpp
+++ b/src/compiler/translator/OutputHLSL.cpp
@@ -1464,7 +1464,8 @@
 
             ensureStructDefined(nodeType);
 
-            out << DecorateUniform(name, nodeType);
+            const TName &nameWithMetadata = node->getName();
+            out << DecorateUniform(nameWithMetadata, nodeType);
         }
         else if (qualifier == EvqAttribute || qualifier == EvqVertexIn)
         {
@@ -1577,6 +1578,42 @@
     }
 }
 
+bool OutputHLSL::ancestorEvaluatesToSamplerInStruct(Visit visit)
+{
+    // Inside InVisit the current node is already in the path.
+    const unsigned int initialN = visit == InVisit ? 1u : 0u;
+    for (unsigned int n = initialN; getAncestorNode(n) != nullptr; ++n)
+    {
+        TIntermNode *ancestor               = getAncestorNode(n);
+        const TIntermBinary *ancestorBinary = ancestor->getAsBinaryNode();
+        if (ancestorBinary == nullptr)
+        {
+            return false;
+        }
+        switch (ancestorBinary->getOp())
+        {
+            case EOpIndexDirectStruct:
+            {
+                const TStructure *structure = ancestorBinary->getLeft()->getType().getStruct();
+                const TIntermConstantUnion *index =
+                    ancestorBinary->getRight()->getAsConstantUnion();
+                const TField *field = structure->fields()[index->getIConst(0)];
+                if (IsSampler(field->type()->getBasicType()))
+                {
+                    return true;
+                }
+                break;
+            }
+            case EOpIndexDirect:
+                break;
+            default:
+                // Returning a sampler from indirect indexing is not supported.
+                return false;
+        }
+    }
+    return false;
+}
+
 bool OutputHLSL::visitBinary(Visit visit, TIntermBinary *node)
 {
     TInfoSinkBase &out = getInfoSink();
@@ -1741,6 +1778,12 @@
                     return false;
                 }
             }
+            else if (ancestorEvaluatesToSamplerInStruct(visit))
+            {
+                // All parts of an expression that access a sampler in a struct need to use _ as
+                // separator to access the sampler variable that has been moved out of the struct.
+                outputTriplet(out, visit, "", "_", "");
+            }
             else
             {
                 outputTriplet(out, visit, "", "[", "]");
@@ -1753,14 +1796,40 @@
         outputTriplet(out, visit, "", "[", "]");
         break;
       case EOpIndexDirectStruct:
-        if (visit == InVisit)
         {
             const TStructure* structure = node->getLeft()->getType().getStruct();
             const TIntermConstantUnion* index = node->getRight()->getAsConstantUnion();
             const TField* field = structure->fields()[index->getIConst(0)];
-            out << "." + DecorateField(field->name(), *structure);
 
-            return false;
+            // In cases where indexing returns a sampler, we need to access the sampler variable
+            // that has been moved out of the struct.
+            bool indexingReturnsSampler = IsSampler(field->type()->getBasicType());
+            if (visit == PreVisit && indexingReturnsSampler)
+            {
+                // Samplers extracted from structs have "angle" prefix to avoid name conflicts.
+                // This prefix is only output at the beginning of the indexing expression, which
+                // may have multiple parts.
+                out << "angle";
+            }
+            if (!indexingReturnsSampler)
+            {
+                // All parts of an expression that access a sampler in a struct need to use _ as
+                // separator to access the sampler variable that has been moved out of the struct.
+                indexingReturnsSampler = ancestorEvaluatesToSamplerInStruct(visit);
+            }
+            if (visit == InVisit)
+            {
+                if (indexingReturnsSampler)
+                {
+                    out << "_" + field->name();
+                }
+                else
+                {
+                    out << "." + DecorateField(field->name(), *structure);
+                }
+
+                return false;
+            }
         }
         break;
       case EOpIndexDirectInterfaceBlock:
@@ -2114,6 +2183,40 @@
     return true;
 }
 
+TString OutputHLSL::samplerNamePrefixFromStruct(TIntermTyped *node)
+{
+    if (node->getAsSymbolNode())
+    {
+        return node->getAsSymbolNode()->getSymbol();
+    }
+    TIntermBinary *nodeBinary = node->getAsBinaryNode();
+    switch (nodeBinary->getOp())
+    {
+        case EOpIndexDirect:
+        {
+            int index = nodeBinary->getRight()->getAsConstantUnion()->getIConst(0);
+
+            TInfoSinkBase prefixSink;
+            prefixSink << samplerNamePrefixFromStruct(nodeBinary->getLeft()) << "_" << index;
+            return TString(prefixSink.c_str());
+        }
+        case EOpIndexDirectStruct:
+        {
+            TStructure *s       = nodeBinary->getLeft()->getAsTyped()->getType().getStruct();
+            int index           = nodeBinary->getRight()->getAsConstantUnion()->getIConst(0);
+            const TField *field = s->fields()[index];
+
+            TInfoSinkBase prefixSink;
+            prefixSink << samplerNamePrefixFromStruct(nodeBinary->getLeft()) << "_"
+                       << field->name();
+            return TString(prefixSink.c_str());
+        }
+        default:
+            UNREACHABLE();
+            return TString("");
+    }
+}
+
 bool OutputHLSL::visitAggregate(Visit visit, TIntermAggregate *node)
 {
     TInfoSinkBase &out = getInfoSink();
@@ -2485,8 +2588,8 @@
 
             for (TIntermSequence::iterator arg = arguments->begin(); arg != arguments->end(); arg++)
             {
-                if (mOutputType == SH_HLSL_4_0_FL9_3_OUTPUT &&
-                    IsSampler((*arg)->getAsTyped()->getBasicType()))
+                TIntermTyped *typedArg = (*arg)->getAsTyped();
+                if (mOutputType == SH_HLSL_4_0_FL9_3_OUTPUT && IsSampler(typedArg->getBasicType()))
                 {
                     out << "texture_";
                     (*arg)->traverse(this);
@@ -2495,6 +2598,30 @@
 
                 (*arg)->traverse(this);
 
+                if (typedArg->getType().isStructureContainingSamplers())
+                {
+                    const TType &argType = typedArg->getType();
+                    TVector<TIntermSymbol *> samplerSymbols;
+                    TString structName = samplerNamePrefixFromStruct(typedArg);
+                    argType.createSamplerSymbols("angle_" + structName, "",
+                                                 argType.isArray() ? argType.getArraySize() : 0,
+                                                 &samplerSymbols, nullptr);
+                    for (const TIntermSymbol *sampler : samplerSymbols)
+                    {
+                        if (mOutputType == SH_HLSL_4_0_FL9_3_OUTPUT)
+                        {
+                            out << ", texture_" << sampler->getSymbol();
+                            out << ", sampler_" << sampler->getSymbol();
+                        }
+                        else
+                        {
+                            // In case of HLSL 4.1+, this symbol is the sampler index, and in case
+                            // of D3D9, it's the sampler variable.
+                            out << ", " + sampler->getSymbol();
+                        }
+                    }
+                }
+
                 if (arg < arguments->end() - 1)
                 {
                     out << ", ";
@@ -3264,7 +3391,49 @@
         }
     }
 
-    return QualifierString(qualifier) + " " + TypeString(type) + " " + nameStr + ArrayString(type);
+    TStringStream argString;
+    argString << QualifierString(qualifier) << " " << TypeString(type) << " " << nameStr
+              << ArrayString(type);
+
+    // If the structure parameter contains samplers, they need to be passed into the function as
+    // separate parameters. HLSL doesn't natively support samplers in structs.
+    if (type.isStructureContainingSamplers())
+    {
+        ASSERT(qualifier != EvqOut && qualifier != EvqInOut);
+        TVector<TIntermSymbol *> samplerSymbols;
+        type.createSamplerSymbols("angle" + nameStr, "", 0, &samplerSymbols, nullptr);
+        for (const TIntermSymbol *sampler : samplerSymbols)
+        {
+            if (mOutputType == SH_HLSL_4_1_OUTPUT)
+            {
+                argString << ", const uint " << sampler->getSymbol() << ArrayString(type);
+            }
+            else if (mOutputType == SH_HLSL_4_0_FL9_3_OUTPUT)
+            {
+                const TType &samplerType = sampler->getType();
+                ASSERT((!type.isArray() && !samplerType.isArray()) ||
+                       type.getArraySize() == samplerType.getArraySize());
+                ASSERT(IsSampler(samplerType.getBasicType()));
+                argString << ", " << QualifierString(qualifier) << " "
+                          << TextureString(samplerType.getBasicType()) << " texture_"
+                          << sampler->getSymbol() << ArrayString(type) << ", "
+                          << QualifierString(qualifier) << " "
+                          << SamplerString(samplerType.getBasicType()) << " sampler_"
+                          << sampler->getSymbol() << ArrayString(type);
+            }
+            else
+            {
+                const TType &samplerType = sampler->getType();
+                ASSERT((!type.isArray() && !samplerType.isArray()) ||
+                       type.getArraySize() == samplerType.getArraySize());
+                ASSERT(IsSampler(samplerType.getBasicType()));
+                argString << ", " << QualifierString(qualifier) << " " << TypeString(samplerType)
+                          << " " << sampler->getSymbol() << ArrayString(type);
+            }
+        }
+    }
+
+    return argString.str();
 }
 
 TString OutputHLSL::initializer(const TType &type)