diff --git a/src/libraries/spirv_cross/NonSemanticShaderDebugInfo100.h b/src/libraries/spirv_cross/NonSemanticShaderDebugInfo100.h new file mode 100644 index 000000000..b276b560c --- /dev/null +++ b/src/libraries/spirv_cross/NonSemanticShaderDebugInfo100.h @@ -0,0 +1,171 @@ +// Copyright (c) 2018-2024 The Khronos Group Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and/or associated documentation files (the "Materials"), +// to deal in the Materials without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Materials, and to permit persons to whom the +// Materials are furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Materials. +// +// MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS KHRONOS +// STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS SPECIFICATIONS AND +// HEADER INFORMATION ARE LOCATED AT https://www.khronos.org/registry/ +// +// THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM,OUT OF OR IN CONNECTION WITH THE MATERIALS OR THE USE OR OTHER DEALINGS +// IN THE MATERIALS. + +#ifndef SPIRV_UNIFIED1_NonSemanticShaderDebugInfo100_H_ +#define SPIRV_UNIFIED1_NonSemanticShaderDebugInfo100_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + NonSemanticShaderDebugInfo100Version = 100, + NonSemanticShaderDebugInfo100Version_BitWidthPadding = 0x7fffffff +}; +enum { + NonSemanticShaderDebugInfo100Revision = 6, + NonSemanticShaderDebugInfo100Revision_BitWidthPadding = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100Instructions { + NonSemanticShaderDebugInfo100DebugInfoNone = 0, + NonSemanticShaderDebugInfo100DebugCompilationUnit = 1, + NonSemanticShaderDebugInfo100DebugTypeBasic = 2, + NonSemanticShaderDebugInfo100DebugTypePointer = 3, + NonSemanticShaderDebugInfo100DebugTypeQualifier = 4, + NonSemanticShaderDebugInfo100DebugTypeArray = 5, + NonSemanticShaderDebugInfo100DebugTypeVector = 6, + NonSemanticShaderDebugInfo100DebugTypedef = 7, + NonSemanticShaderDebugInfo100DebugTypeFunction = 8, + NonSemanticShaderDebugInfo100DebugTypeEnum = 9, + NonSemanticShaderDebugInfo100DebugTypeComposite = 10, + NonSemanticShaderDebugInfo100DebugTypeMember = 11, + NonSemanticShaderDebugInfo100DebugTypeInheritance = 12, + NonSemanticShaderDebugInfo100DebugTypePtrToMember = 13, + NonSemanticShaderDebugInfo100DebugTypeTemplate = 14, + NonSemanticShaderDebugInfo100DebugTypeTemplateParameter = 15, + NonSemanticShaderDebugInfo100DebugTypeTemplateTemplateParameter = 16, + NonSemanticShaderDebugInfo100DebugTypeTemplateParameterPack = 17, + NonSemanticShaderDebugInfo100DebugGlobalVariable = 18, + NonSemanticShaderDebugInfo100DebugFunctionDeclaration = 19, + NonSemanticShaderDebugInfo100DebugFunction = 20, + NonSemanticShaderDebugInfo100DebugLexicalBlock = 21, + NonSemanticShaderDebugInfo100DebugLexicalBlockDiscriminator = 22, + NonSemanticShaderDebugInfo100DebugScope = 23, + NonSemanticShaderDebugInfo100DebugNoScope = 24, + NonSemanticShaderDebugInfo100DebugInlinedAt = 25, + NonSemanticShaderDebugInfo100DebugLocalVariable = 26, + NonSemanticShaderDebugInfo100DebugInlinedVariable = 27, + NonSemanticShaderDebugInfo100DebugDeclare = 28, + NonSemanticShaderDebugInfo100DebugValue = 29, + NonSemanticShaderDebugInfo100DebugOperation = 30, + NonSemanticShaderDebugInfo100DebugExpression = 31, + NonSemanticShaderDebugInfo100DebugMacroDef = 32, + NonSemanticShaderDebugInfo100DebugMacroUndef = 33, + NonSemanticShaderDebugInfo100DebugImportedEntity = 34, + NonSemanticShaderDebugInfo100DebugSource = 35, + NonSemanticShaderDebugInfo100DebugFunctionDefinition = 101, + NonSemanticShaderDebugInfo100DebugSourceContinued = 102, + NonSemanticShaderDebugInfo100DebugLine = 103, + NonSemanticShaderDebugInfo100DebugNoLine = 104, + NonSemanticShaderDebugInfo100DebugBuildIdentifier = 105, + NonSemanticShaderDebugInfo100DebugStoragePath = 106, + NonSemanticShaderDebugInfo100DebugEntryPoint = 107, + NonSemanticShaderDebugInfo100DebugTypeMatrix = 108, + NonSemanticShaderDebugInfo100InstructionsMax = 0x7fffffff +}; + + +enum NonSemanticShaderDebugInfo100DebugInfoFlags { + NonSemanticShaderDebugInfo100None = 0x0000, + NonSemanticShaderDebugInfo100FlagIsProtected = 0x01, + NonSemanticShaderDebugInfo100FlagIsPrivate = 0x02, + NonSemanticShaderDebugInfo100FlagIsPublic = 0x03, + NonSemanticShaderDebugInfo100FlagIsLocal = 0x04, + NonSemanticShaderDebugInfo100FlagIsDefinition = 0x08, + NonSemanticShaderDebugInfo100FlagFwdDecl = 0x10, + NonSemanticShaderDebugInfo100FlagArtificial = 0x20, + NonSemanticShaderDebugInfo100FlagExplicit = 0x40, + NonSemanticShaderDebugInfo100FlagPrototyped = 0x80, + NonSemanticShaderDebugInfo100FlagObjectPointer = 0x100, + NonSemanticShaderDebugInfo100FlagStaticMember = 0x200, + NonSemanticShaderDebugInfo100FlagIndirectVariable = 0x400, + NonSemanticShaderDebugInfo100FlagLValueReference = 0x800, + NonSemanticShaderDebugInfo100FlagRValueReference = 0x1000, + NonSemanticShaderDebugInfo100FlagIsOptimized = 0x2000, + NonSemanticShaderDebugInfo100FlagIsEnumClass = 0x4000, + NonSemanticShaderDebugInfo100FlagTypePassByValue = 0x8000, + NonSemanticShaderDebugInfo100FlagTypePassByReference = 0x10000, + NonSemanticShaderDebugInfo100FlagUnknownPhysicalLayout = 0x20000, + NonSemanticShaderDebugInfo100DebugInfoFlagsMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100BuildIdentifierFlags { + NonSemanticShaderDebugInfo100IdentifierPossibleDuplicates = 0x01, + NonSemanticShaderDebugInfo100BuildIdentifierFlagsMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100DebugBaseTypeAttributeEncoding { + NonSemanticShaderDebugInfo100Unspecified = 0, + NonSemanticShaderDebugInfo100Address = 1, + NonSemanticShaderDebugInfo100Boolean = 2, + NonSemanticShaderDebugInfo100Float = 3, + NonSemanticShaderDebugInfo100Signed = 4, + NonSemanticShaderDebugInfo100SignedChar = 5, + NonSemanticShaderDebugInfo100Unsigned = 6, + NonSemanticShaderDebugInfo100UnsignedChar = 7, + NonSemanticShaderDebugInfo100DebugBaseTypeAttributeEncodingMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100DebugCompositeType { + NonSemanticShaderDebugInfo100Class = 0, + NonSemanticShaderDebugInfo100Structure = 1, + NonSemanticShaderDebugInfo100Union = 2, + NonSemanticShaderDebugInfo100DebugCompositeTypeMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100DebugTypeQualifier { + NonSemanticShaderDebugInfo100ConstType = 0, + NonSemanticShaderDebugInfo100VolatileType = 1, + NonSemanticShaderDebugInfo100RestrictType = 2, + NonSemanticShaderDebugInfo100AtomicType = 3, + NonSemanticShaderDebugInfo100DebugTypeQualifierMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100DebugOperation { + NonSemanticShaderDebugInfo100Deref = 0, + NonSemanticShaderDebugInfo100Plus = 1, + NonSemanticShaderDebugInfo100Minus = 2, + NonSemanticShaderDebugInfo100PlusUconst = 3, + NonSemanticShaderDebugInfo100BitPiece = 4, + NonSemanticShaderDebugInfo100Swap = 5, + NonSemanticShaderDebugInfo100Xderef = 6, + NonSemanticShaderDebugInfo100StackValue = 7, + NonSemanticShaderDebugInfo100Constu = 8, + NonSemanticShaderDebugInfo100Fragment = 9, + NonSemanticShaderDebugInfo100DebugOperationMax = 0x7fffffff +}; + +enum NonSemanticShaderDebugInfo100DebugImportedEntity { + NonSemanticShaderDebugInfo100ImportedModule = 0, + NonSemanticShaderDebugInfo100ImportedDeclaration = 1, + NonSemanticShaderDebugInfo100DebugImportedEntityMax = 0x7fffffff +}; + + +#ifdef __cplusplus +} +#endif + +#endif // SPIRV_UNIFIED1_NonSemanticShaderDebugInfo100_H_ diff --git a/src/libraries/spirv_cross/spirv.hpp b/src/libraries/spirv_cross/spirv.hpp index e25264af2..462c3f6bf 100644 --- a/src/libraries/spirv_cross/spirv.hpp +++ b/src/libraries/spirv_cross/spirv.hpp @@ -1,26 +1,10 @@ -// Copyright (c) 2014-2020 The Khronos Group Inc. +// Copyright: 2014-2024 The Khronos Group Inc. +// License: MIT // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and/or associated documentation files (the "Materials"), -// to deal in the Materials without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Materials, and to permit persons to whom the -// Materials are furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Materials. -// -// MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS KHRONOS -// STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS SPECIFICATIONS AND -// HEADER INFORMATION ARE LOCATED AT https://www.khronos.org/registry/ -// -// THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM,OUT OF OR IN CONNECTION WITH THE MATERIALS OR THE USE OR OTHER DEALINGS -// IN THE MATERIALS. +// MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS +// KHRONOS STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS +// SPECIFICATIONS AND HEADER INFORMATION ARE LOCATED AT +// https://www.khronos.org/registry/ // This header is automatically generated by the same tool that creates // the Binary Section of the SPIR-V specification. @@ -69,6 +53,12 @@ enum SourceLanguage { SourceLanguageHLSL = 5, SourceLanguageCPP_for_OpenCL = 6, SourceLanguageSYCL = 7, + SourceLanguageHERO_C = 8, + SourceLanguageNZSL = 9, + SourceLanguageWGSL = 10, + SourceLanguageSlang = 11, + SourceLanguageZig = 12, + SourceLanguageRust = 13, SourceLanguageMax = 0x7fffffff, }; @@ -156,6 +146,9 @@ enum ExecutionMode { ExecutionModeSubgroupsPerWorkgroupId = 37, ExecutionModeLocalSizeId = 38, ExecutionModeLocalSizeHintId = 39, + ExecutionModeNonCoherentColorAttachmentReadEXT = 4169, + ExecutionModeNonCoherentDepthAttachmentReadEXT = 4170, + ExecutionModeNonCoherentStencilAttachmentReadEXT = 4171, ExecutionModeSubgroupUniformControlFlowKHR = 4421, ExecutionModePostDepthCoverage = 4446, ExecutionModeDenormPreserve = 4459, @@ -163,19 +156,33 @@ enum ExecutionMode { ExecutionModeSignedZeroInfNanPreserve = 4461, ExecutionModeRoundingModeRTE = 4462, ExecutionModeRoundingModeRTZ = 4463, + ExecutionModeNonCoherentTileAttachmentReadQCOM = 4489, + ExecutionModeTileShadingRateQCOM = 4490, ExecutionModeEarlyAndLateFragmentTestsAMD = 5017, ExecutionModeStencilRefReplacingEXT = 5027, + ExecutionModeCoalescingAMDX = 5069, + ExecutionModeIsApiEntryAMDX = 5070, + ExecutionModeMaxNodeRecursionAMDX = 5071, + ExecutionModeStaticNumWorkgroupsAMDX = 5072, + ExecutionModeShaderIndexAMDX = 5073, + ExecutionModeMaxNumWorkgroupsAMDX = 5077, ExecutionModeStencilRefUnchangedFrontAMD = 5079, ExecutionModeStencilRefGreaterFrontAMD = 5080, ExecutionModeStencilRefLessFrontAMD = 5081, ExecutionModeStencilRefUnchangedBackAMD = 5082, ExecutionModeStencilRefGreaterBackAMD = 5083, ExecutionModeStencilRefLessBackAMD = 5084, + ExecutionModeQuadDerivativesKHR = 5088, + ExecutionModeRequireFullQuadsKHR = 5089, + ExecutionModeSharesInputWithAMDX = 5102, + ExecutionModeArithmeticPoisonKHR = 5157, ExecutionModeOutputLinesEXT = 5269, ExecutionModeOutputLinesNV = 5269, ExecutionModeOutputPrimitivesEXT = 5270, ExecutionModeOutputPrimitivesNV = 5270, + ExecutionModeDerivativeGroupQuadsKHR = 5289, ExecutionModeDerivativeGroupQuadsNV = 5289, + ExecutionModeDerivativeGroupLinearKHR = 5290, ExecutionModeDerivativeGroupLinearNV = 5290, ExecutionModeOutputTrianglesEXT = 5298, ExecutionModeOutputTrianglesNV = 5298, @@ -185,6 +192,7 @@ enum ExecutionMode { ExecutionModeSampleInterlockUnorderedEXT = 5369, ExecutionModeShadingRateInterlockOrderedEXT = 5370, ExecutionModeShadingRateInterlockUnorderedEXT = 5371, + ExecutionModeShader64BitIndexingEXT = 5427, ExecutionModeSharedLocalMemorySizeINTEL = 5618, ExecutionModeRoundingModeRTPINTEL = 5620, ExecutionModeRoundingModeRTNINTEL = 5621, @@ -195,7 +203,14 @@ enum ExecutionMode { ExecutionModeNoGlobalOffsetINTEL = 5895, ExecutionModeNumSIMDWorkitemsINTEL = 5896, ExecutionModeSchedulerTargetFmaxMhzINTEL = 5903, + ExecutionModeMaximallyReconvergesKHR = 6023, + ExecutionModeFPFastMathDefault = 6028, + ExecutionModeStreamingInterfaceINTEL = 6154, + ExecutionModeRegisterMapInterfaceINTEL = 6160, ExecutionModeNamedBarrierCountINTEL = 6417, + ExecutionModeMaximumRegistersINTEL = 6461, + ExecutionModeMaximumRegistersIdINTEL = 6462, + ExecutionModeNamedMaximumRegistersINTEL = 6463, ExecutionModeMax = 0x7fffffff, }; @@ -213,6 +228,9 @@ enum StorageClass { StorageClassAtomicCounter = 10, StorageClassImage = 11, StorageClassStorageBuffer = 12, + StorageClassTileImageEXT = 4172, + StorageClassTileAttachmentQCOM = 4491, + StorageClassNodePayloadAMDX = 5068, StorageClassCallableDataKHR = 5328, StorageClassCallableDataNV = 5328, StorageClassIncomingCallableDataKHR = 5329, @@ -227,9 +245,13 @@ enum StorageClass { StorageClassShaderRecordBufferNV = 5343, StorageClassPhysicalStorageBuffer = 5349, StorageClassPhysicalStorageBufferEXT = 5349, + StorageClassHitObjectAttributeNV = 5385, StorageClassTaskPayloadWorkgroupEXT = 5402, + StorageClassHitObjectAttributeEXT = 5411, StorageClassCodeSectionINTEL = 5605, + StorageClassDeviceOnlyALTERA = 5936, StorageClassDeviceOnlyINTEL = 5936, + StorageClassHostOnlyALTERA = 5937, StorageClassHostOnlyINTEL = 5937, StorageClassMax = 0x7fffffff, }; @@ -242,6 +264,7 @@ enum Dim { DimRect = 4, DimBuffer = 5, DimSubpassData = 6, + DimTileImageDataEXT = 4173, DimMax = 0x7fffffff, }; @@ -348,6 +371,15 @@ enum ImageChannelDataType { ImageChannelDataTypeFloat = 14, ImageChannelDataTypeUnormInt24 = 15, ImageChannelDataTypeUnormInt101010_2 = 16, + ImageChannelDataTypeUnormInt10X6EXT = 17, + ImageChannelDataTypeUnsignedIntRaw10EXT = 19, + ImageChannelDataTypeUnsignedIntRaw12EXT = 20, + ImageChannelDataTypeUnormInt2_101010EXT = 21, + ImageChannelDataTypeUnsignedInt10X6EXT = 22, + ImageChannelDataTypeUnsignedInt12X4EXT = 23, + ImageChannelDataTypeUnsignedInt14X2EXT = 24, + ImageChannelDataTypeUnormInt12X4EXT = 25, + ImageChannelDataTypeUnormInt14X2EXT = 26, ImageChannelDataTypeMax = 0x7fffffff, }; @@ -405,8 +437,11 @@ enum FPFastMathModeShift { FPFastMathModeNSZShift = 2, FPFastMathModeAllowRecipShift = 3, FPFastMathModeFastShift = 4, + FPFastMathModeAllowContractShift = 16, FPFastMathModeAllowContractFastINTELShift = 16, + FPFastMathModeAllowReassocShift = 17, FPFastMathModeAllowReassocINTELShift = 17, + FPFastMathModeAllowTransformShift = 18, FPFastMathModeMax = 0x7fffffff, }; @@ -417,8 +452,11 @@ enum FPFastMathModeMask { FPFastMathModeNSZMask = 0x00000004, FPFastMathModeAllowRecipMask = 0x00000008, FPFastMathModeFastMask = 0x00000010, + FPFastMathModeAllowContractMask = 0x00010000, FPFastMathModeAllowContractFastINTELMask = 0x00010000, + FPFastMathModeAllowReassocMask = 0x00020000, FPFastMathModeAllowReassocINTELMask = 0x00020000, + FPFastMathModeAllowTransformMask = 0x00040000, }; enum FPRoundingMode { @@ -452,6 +490,8 @@ enum FunctionParameterAttribute { FunctionParameterAttributeNoCapture = 5, FunctionParameterAttributeNoWrite = 6, FunctionParameterAttributeNoReadWrite = 7, + FunctionParameterAttributeRuntimeAlignedALTERA = 5940, + FunctionParameterAttributeRuntimeAlignedINTEL = 5940, FunctionParameterAttributeMax = 0x7fffffff, }; @@ -503,9 +543,24 @@ enum Decoration { DecorationMaxByteOffset = 45, DecorationAlignmentId = 46, DecorationMaxByteOffsetId = 47, + DecorationSaturatedToLargestFloat8NormalConversionEXT = 4216, DecorationNoSignedWrap = 4469, DecorationNoUnsignedWrap = 4470, + DecorationWeightTextureQCOM = 4487, + DecorationBlockMatchTextureQCOM = 4488, + DecorationBlockMatchSamplerQCOM = 4499, DecorationExplicitInterpAMD = 4999, + DecorationNodeSharesPayloadLimitsWithAMDX = 5019, + DecorationNodeMaxPayloadsAMDX = 5020, + DecorationTrackFinishWritingAMDX = 5078, + DecorationPayloadNodeNameAMDX = 5091, + DecorationPayloadNodeBaseIndexAMDX = 5098, + DecorationPayloadNodeSparseArrayAMDX = 5099, + DecorationPayloadNodeArraySizeAMDX = 5100, + DecorationPayloadDispatchIndirectAMDX = 5105, + DecorationArrayStrideIdEXT = 5124, + DecorationOffsetIdEXT = 5125, + DecorationUTFEncodedKHR = 5145, DecorationOverrideCoverageNV = 5248, DecorationPassthroughNV = 5250, DecorationViewportRelativeNV = 5252, @@ -522,6 +577,10 @@ enum Decoration { DecorationRestrictPointerEXT = 5355, DecorationAliasedPointer = 5356, DecorationAliasedPointerEXT = 5356, + DecorationMemberOffsetNV = 5358, + DecorationHitObjectShaderRecordBufferNV = 5386, + DecorationHitObjectShaderRecordBufferEXT = 5389, + DecorationBankNV = 5397, DecorationBindlessSamplerNV = 5398, DecorationBindlessImageNV = 5399, DecorationBoundSamplerNV = 5400, @@ -542,32 +601,99 @@ enum Decoration { DecorationUserTypeGOOGLE = 5636, DecorationFunctionRoundingModeINTEL = 5822, DecorationFunctionDenormModeINTEL = 5823, + DecorationRegisterALTERA = 5825, DecorationRegisterINTEL = 5825, + DecorationMemoryALTERA = 5826, DecorationMemoryINTEL = 5826, + DecorationNumbanksALTERA = 5827, DecorationNumbanksINTEL = 5827, + DecorationBankwidthALTERA = 5828, DecorationBankwidthINTEL = 5828, + DecorationMaxPrivateCopiesALTERA = 5829, DecorationMaxPrivateCopiesINTEL = 5829, + DecorationSinglepumpALTERA = 5830, DecorationSinglepumpINTEL = 5830, + DecorationDoublepumpALTERA = 5831, DecorationDoublepumpINTEL = 5831, + DecorationMaxReplicatesALTERA = 5832, DecorationMaxReplicatesINTEL = 5832, + DecorationSimpleDualPortALTERA = 5833, DecorationSimpleDualPortINTEL = 5833, + DecorationMergeALTERA = 5834, DecorationMergeINTEL = 5834, + DecorationBankBitsALTERA = 5835, DecorationBankBitsINTEL = 5835, + DecorationForcePow2DepthALTERA = 5836, DecorationForcePow2DepthINTEL = 5836, + DecorationStridesizeALTERA = 5883, + DecorationStridesizeINTEL = 5883, + DecorationWordsizeALTERA = 5884, + DecorationWordsizeINTEL = 5884, + DecorationTrueDualPortALTERA = 5885, + DecorationTrueDualPortINTEL = 5885, + DecorationBurstCoalesceALTERA = 5899, DecorationBurstCoalesceINTEL = 5899, + DecorationCacheSizeALTERA = 5900, DecorationCacheSizeINTEL = 5900, + DecorationDontStaticallyCoalesceALTERA = 5901, DecorationDontStaticallyCoalesceINTEL = 5901, + DecorationPrefetchALTERA = 5902, DecorationPrefetchINTEL = 5902, + DecorationStallEnableALTERA = 5905, DecorationStallEnableINTEL = 5905, + DecorationFuseLoopsInFunctionALTERA = 5907, DecorationFuseLoopsInFunctionINTEL = 5907, + DecorationMathOpDSPModeALTERA = 5909, + DecorationMathOpDSPModeINTEL = 5909, DecorationAliasScopeINTEL = 5914, DecorationNoAliasINTEL = 5915, + DecorationInitiationIntervalALTERA = 5917, + DecorationInitiationIntervalINTEL = 5917, + DecorationMaxConcurrencyALTERA = 5918, + DecorationMaxConcurrencyINTEL = 5918, + DecorationPipelineEnableALTERA = 5919, + DecorationPipelineEnableINTEL = 5919, + DecorationBufferLocationALTERA = 5921, DecorationBufferLocationINTEL = 5921, + DecorationIOPipeStorageALTERA = 5944, DecorationIOPipeStorageINTEL = 5944, DecorationFunctionFloatingPointModeINTEL = 6080, DecorationSingleElementVectorINTEL = 6085, DecorationVectorComputeCallableFunctionINTEL = 6087, DecorationMediaBlockIOINTEL = 6140, + DecorationStallFreeALTERA = 6151, + DecorationStallFreeINTEL = 6151, + DecorationFPMaxErrorDecorationINTEL = 6170, + DecorationLatencyControlLabelALTERA = 6172, + DecorationLatencyControlLabelINTEL = 6172, + DecorationLatencyControlConstraintALTERA = 6173, + DecorationLatencyControlConstraintINTEL = 6173, + DecorationConduitKernelArgumentALTERA = 6175, + DecorationConduitKernelArgumentINTEL = 6175, + DecorationRegisterMapKernelArgumentALTERA = 6176, + DecorationRegisterMapKernelArgumentINTEL = 6176, + DecorationMMHostInterfaceAddressWidthALTERA = 6177, + DecorationMMHostInterfaceAddressWidthINTEL = 6177, + DecorationMMHostInterfaceDataWidthALTERA = 6178, + DecorationMMHostInterfaceDataWidthINTEL = 6178, + DecorationMMHostInterfaceLatencyALTERA = 6179, + DecorationMMHostInterfaceLatencyINTEL = 6179, + DecorationMMHostInterfaceReadWriteModeALTERA = 6180, + DecorationMMHostInterfaceReadWriteModeINTEL = 6180, + DecorationMMHostInterfaceMaxBurstALTERA = 6181, + DecorationMMHostInterfaceMaxBurstINTEL = 6181, + DecorationMMHostInterfaceWaitRequestALTERA = 6182, + DecorationMMHostInterfaceWaitRequestINTEL = 6182, + DecorationStableKernelArgumentALTERA = 6183, + DecorationStableKernelArgumentINTEL = 6183, + DecorationHostAccessINTEL = 6188, + DecorationInitModeALTERA = 6190, + DecorationInitModeINTEL = 6190, + DecorationImplementInRegisterMapALTERA = 6191, + DecorationImplementInRegisterMapINTEL = 6191, + DecorationConditionalINTEL = 6247, + DecorationCacheControlLoadINTEL = 6442, + DecorationCacheControlStoreINTEL = 6443, DecorationMax = 0x7fffffff, }; @@ -613,6 +739,11 @@ enum BuiltIn { BuiltInSubgroupLocalInvocationId = 41, BuiltInVertexIndex = 42, BuiltInInstanceIndex = 43, + BuiltInCoreIDARM = 4160, + BuiltInCoreCountARM = 4161, + BuiltInCoreMaxIDARM = 4162, + BuiltInWarpIDARM = 4163, + BuiltInWarpMaxIDARM = 4164, BuiltInSubgroupEqMask = 4416, BuiltInSubgroupEqMaskKHR = 4416, BuiltInSubgroupGeMask = 4417, @@ -630,6 +761,9 @@ enum BuiltIn { BuiltInDeviceIndex = 4438, BuiltInViewIndex = 4440, BuiltInShadingRateKHR = 4444, + BuiltInTileOffsetQCOM = 4492, + BuiltInTileDimensionQCOM = 4493, + BuiltInTileApronSizeQCOM = 4494, BuiltInBaryCoordNoPerspAMD = 4992, BuiltInBaryCoordNoPerspCentroidAMD = 4993, BuiltInBaryCoordNoPerspSampleAMD = 4994, @@ -638,6 +772,10 @@ enum BuiltIn { BuiltInBaryCoordSmoothSampleAMD = 4997, BuiltInBaryCoordPullModelAMD = 4998, BuiltInFragStencilRefEXT = 5014, + BuiltInRemainingRecursionLevelsAMDX = 5021, + BuiltInShaderIndexAMDX = 5073, + BuiltInSamplerHeapEXT = 5122, + BuiltInResourceHeapEXT = 5123, BuiltInViewportMaskNV = 5253, BuiltInSecondaryPositionNV = 5257, BuiltInSecondaryViewportMaskNV = 5258, @@ -690,13 +828,25 @@ enum BuiltIn { BuiltInHitKindKHR = 5333, BuiltInHitKindNV = 5333, BuiltInCurrentRayTimeNV = 5334, + BuiltInHitTriangleVertexPositionsKHR = 5335, + BuiltInHitMicroTriangleVertexPositionsNV = 5337, + BuiltInHitMicroTriangleVertexBarycentricsNV = 5344, BuiltInIncomingRayFlagsKHR = 5351, BuiltInIncomingRayFlagsNV = 5351, BuiltInRayGeometryIndexKHR = 5352, + BuiltInHitIsSphereNV = 5359, + BuiltInHitIsLSSNV = 5360, + BuiltInHitSpherePositionNV = 5361, BuiltInWarpsPerSMNV = 5374, BuiltInSMCountNV = 5375, BuiltInWarpIDNV = 5376, BuiltInSMIDNV = 5377, + BuiltInHitLSSPositionsNV = 5396, + BuiltInHitKindFrontFacingMicroTriangleNV = 5405, + BuiltInHitKindBackFacingMicroTriangleNV = 5406, + BuiltInHitSphereRadiusNV = 5420, + BuiltInHitLSSRadiiNV = 5421, + BuiltInClusterIDNV = 5436, BuiltInCullMaskKHR = 6021, BuiltInMax = 0x7fffffff, }; @@ -723,14 +873,26 @@ enum LoopControlShift { LoopControlIterationMultipleShift = 6, LoopControlPeelCountShift = 7, LoopControlPartialCountShift = 8, + LoopControlInitiationIntervalALTERAShift = 16, LoopControlInitiationIntervalINTELShift = 16, + LoopControlMaxConcurrencyALTERAShift = 17, LoopControlMaxConcurrencyINTELShift = 17, + LoopControlDependencyArrayALTERAShift = 18, LoopControlDependencyArrayINTELShift = 18, + LoopControlPipelineEnableALTERAShift = 19, LoopControlPipelineEnableINTELShift = 19, + LoopControlLoopCoalesceALTERAShift = 20, LoopControlLoopCoalesceINTELShift = 20, + LoopControlMaxInterleavingALTERAShift = 21, LoopControlMaxInterleavingINTELShift = 21, + LoopControlSpeculatedIterationsALTERAShift = 22, LoopControlSpeculatedIterationsINTELShift = 22, + LoopControlNoFusionALTERAShift = 23, LoopControlNoFusionINTELShift = 23, + LoopControlLoopCountALTERAShift = 24, + LoopControlLoopCountINTELShift = 24, + LoopControlMaxReinvocationDelayALTERAShift = 25, + LoopControlMaxReinvocationDelayINTELShift = 25, LoopControlMax = 0x7fffffff, }; @@ -745,14 +907,26 @@ enum LoopControlMask { LoopControlIterationMultipleMask = 0x00000040, LoopControlPeelCountMask = 0x00000080, LoopControlPartialCountMask = 0x00000100, + LoopControlInitiationIntervalALTERAMask = 0x00010000, LoopControlInitiationIntervalINTELMask = 0x00010000, + LoopControlMaxConcurrencyALTERAMask = 0x00020000, LoopControlMaxConcurrencyINTELMask = 0x00020000, + LoopControlDependencyArrayALTERAMask = 0x00040000, LoopControlDependencyArrayINTELMask = 0x00040000, + LoopControlPipelineEnableALTERAMask = 0x00080000, LoopControlPipelineEnableINTELMask = 0x00080000, + LoopControlLoopCoalesceALTERAMask = 0x00100000, LoopControlLoopCoalesceINTELMask = 0x00100000, + LoopControlMaxInterleavingALTERAMask = 0x00200000, LoopControlMaxInterleavingINTELMask = 0x00200000, + LoopControlSpeculatedIterationsALTERAMask = 0x00400000, LoopControlSpeculatedIterationsINTELMask = 0x00400000, + LoopControlNoFusionALTERAMask = 0x00800000, LoopControlNoFusionINTELMask = 0x00800000, + LoopControlLoopCountALTERAMask = 0x01000000, + LoopControlLoopCountINTELMask = 0x01000000, + LoopControlMaxReinvocationDelayALTERAMask = 0x02000000, + LoopControlMaxReinvocationDelayINTELMask = 0x02000000, }; enum FunctionControlShift { @@ -760,6 +934,7 @@ enum FunctionControlShift { FunctionControlDontInlineShift = 1, FunctionControlPureShift = 2, FunctionControlConstShift = 3, + FunctionControlOptNoneEXTShift = 16, FunctionControlOptNoneINTELShift = 16, FunctionControlMax = 0x7fffffff, }; @@ -770,6 +945,7 @@ enum FunctionControlMask { FunctionControlDontInlineMask = 0x00000002, FunctionControlPureMask = 0x00000004, FunctionControlConstMask = 0x00000008, + FunctionControlOptNoneEXTMask = 0x00010000, FunctionControlOptNoneINTELMask = 0x00010000, }; @@ -862,8 +1038,11 @@ enum GroupOperation { GroupOperationInclusiveScan = 1, GroupOperationExclusiveScan = 2, GroupOperationClusteredReduce = 3, + GroupOperationPartitionedReduceEXT = 6, GroupOperationPartitionedReduceNV = 6, + GroupOperationPartitionedInclusiveScanEXT = 7, GroupOperationPartitionedInclusiveScanNV = 7, + GroupOperationPartitionedExclusiveScanEXT = 8, GroupOperationPartitionedExclusiveScanNV = 8, GroupOperationMax = 0x7fffffff, }; @@ -956,6 +1135,17 @@ enum Capability { CapabilityShaderLayer = 69, CapabilityShaderViewportIndex = 70, CapabilityUniformDecoration = 71, + CapabilityCoreBuiltinsARM = 4165, + CapabilityTileImageColorReadAccessEXT = 4166, + CapabilityTileImageDepthReadAccessEXT = 4167, + CapabilityTileImageStencilReadAccessEXT = 4168, + CapabilityTensorsARM = 4174, + CapabilityStorageTensorArrayDynamicIndexingARM = 4175, + CapabilityStorageTensorArrayNonUniformIndexingARM = 4176, + CapabilityGraphARM = 4191, + CapabilityCooperativeMatrixLayoutsARM = 4201, + CapabilityFloat8EXT = 4212, + CapabilityFloat8CooperativeMatrixEXT = 4213, CapabilityFragmentShadingRateKHR = 4422, CapabilitySubgroupBallotKHR = 4423, CapabilityDrawParameters = 4427, @@ -985,8 +1175,15 @@ enum Capability { CapabilityRoundingModeRTZ = 4468, CapabilityRayQueryProvisionalKHR = 4471, CapabilityRayQueryKHR = 4472, + CapabilityUntypedPointersKHR = 4473, CapabilityRayTraversalPrimitiveCullingKHR = 4478, CapabilityRayTracingKHR = 4479, + CapabilityTextureSampleWeightedQCOM = 4484, + CapabilityTextureBoxFilterQCOM = 4485, + CapabilityTextureBlockMatchQCOM = 4486, + CapabilityTileShadingQCOM = 4495, + CapabilityCooperativeMatrixConversionQCOM = 4496, + CapabilityTextureBlockMatch2QCOM = 4498, CapabilityFloat16ImageAMD = 5008, CapabilityImageGatherBiasLodAMD = 5009, CapabilityFragmentMaskAMD = 5010, @@ -994,6 +1191,17 @@ enum Capability { CapabilityImageReadWriteLodAMD = 5015, CapabilityInt64ImageEXT = 5016, CapabilityShaderClockKHR = 5055, + CapabilityShaderEnqueueAMDX = 5067, + CapabilityQuadControlKHR = 5087, + CapabilityInt4TypeINTEL = 5112, + CapabilityInt4CooperativeMatrixINTEL = 5114, + CapabilityBFloat16TypeKHR = 5116, + CapabilityBFloat16DotProductKHR = 5117, + CapabilityBFloat16CooperativeMatrixKHR = 5118, + CapabilityAbortKHR = 5120, + CapabilityDescriptorHeapEXT = 5128, + CapabilityConstantDataKHR = 5146, + CapabilityPoisonFreezeKHR = 5156, CapabilitySampleMaskOverrideCoverageNV = 5249, CapabilityGeometryShaderPassthroughNV = 5251, CapabilityShaderViewportIndexLayerEXT = 5254, @@ -1007,9 +1215,11 @@ enum Capability { CapabilityMeshShadingEXT = 5283, CapabilityFragmentBarycentricKHR = 5284, CapabilityFragmentBarycentricNV = 5284, + CapabilityComputeDerivativeGroupQuadsKHR = 5288, CapabilityComputeDerivativeGroupQuadsNV = 5288, CapabilityFragmentDensityEXT = 5291, CapabilityShadingRateNV = 5291, + CapabilityGroupNonUniformPartitionedEXT = 5297, CapabilityGroupNonUniformPartitionedNV = 5297, CapabilityShaderNonUniform = 5301, CapabilityShaderNonUniformEXT = 5301, @@ -1035,6 +1245,7 @@ enum Capability { CapabilityUniformTexelBufferArrayNonUniformIndexingEXT = 5311, CapabilityStorageTexelBufferArrayNonUniformIndexing = 5312, CapabilityStorageTexelBufferArrayNonUniformIndexingEXT = 5312, + CapabilityRayTracingPositionFetchKHR = 5336, CapabilityRayTracingNV = 5340, CapabilityRayTracingMotionBlurNV = 5341, CapabilityVulkanMemoryModel = 5345, @@ -1043,6 +1254,7 @@ enum Capability { CapabilityVulkanMemoryModelDeviceScopeKHR = 5346, CapabilityPhysicalStorageBufferAddresses = 5347, CapabilityPhysicalStorageBufferAddressesEXT = 5347, + CapabilityComputeDerivativeGroupLinearKHR = 5350, CapabilityComputeDerivativeGroupLinearNV = 5350, CapabilityRayTracingProvisionalKHR = 5353, CapabilityCooperativeMatrixNV = 5357, @@ -1052,7 +1264,29 @@ enum Capability { CapabilityFragmentShaderPixelInterlockEXT = 5378, CapabilityDemoteToHelperInvocation = 5379, CapabilityDemoteToHelperInvocationEXT = 5379, + CapabilityDisplacementMicromapNV = 5380, + CapabilityRayTracingOpacityMicromapEXT = 5381, + CapabilityShaderInvocationReorderNV = 5383, + CapabilityShaderInvocationReorderEXT = 5388, CapabilityBindlessTextureNV = 5390, + CapabilityRayQueryPositionFetchKHR = 5391, + CapabilityCooperativeVectorNV = 5394, + CapabilityAtomicFloat16VectorNV = 5404, + CapabilityRayTracingDisplacementMicromapNV = 5409, + CapabilityRawAccessChainsNV = 5414, + CapabilityRayTracingSpheresGeometryNV = 5418, + CapabilityRayTracingLinearSweptSpheresGeometryNV = 5419, + CapabilityPushConstantBanksNV = 5423, + CapabilityLongVectorEXT = 5425, + CapabilityShader64BitIndexingEXT = 5426, + CapabilityCooperativeMatrixReductionsNV = 5430, + CapabilityCooperativeMatrixConversionsNV = 5431, + CapabilityCooperativeMatrixPerElementOperationsNV = 5432, + CapabilityCooperativeMatrixTensorAddressingNV = 5433, + CapabilityCooperativeMatrixBlockLoadsNV = 5434, + CapabilityCooperativeVectorTrainingNV = 5435, + CapabilityRayTracingClusterAccelerationStructureNV = 5437, + CapabilityTensorAddressingNV = 5439, CapabilitySubgroupShuffleINTEL = 5568, CapabilitySubgroupBufferBlockIOINTEL = 5569, CapabilitySubgroupImageBlockIOINTEL = 5570, @@ -1074,23 +1308,42 @@ enum Capability { CapabilitySubgroupAvcMotionEstimationChromaINTEL = 5698, CapabilityVariableLengthArrayINTEL = 5817, CapabilityFunctionFloatControlINTEL = 5821, + CapabilityFPGAMemoryAttributesALTERA = 5824, CapabilityFPGAMemoryAttributesINTEL = 5824, CapabilityFPFastMathModeINTEL = 5837, + CapabilityArbitraryPrecisionIntegersALTERA = 5844, CapabilityArbitraryPrecisionIntegersINTEL = 5844, + CapabilityArbitraryPrecisionFloatingPointALTERA = 5845, CapabilityArbitraryPrecisionFloatingPointINTEL = 5845, CapabilityUnstructuredLoopControlsINTEL = 5886, + CapabilityFPGALoopControlsALTERA = 5888, CapabilityFPGALoopControlsINTEL = 5888, CapabilityKernelAttributesINTEL = 5892, CapabilityFPGAKernelAttributesINTEL = 5897, + CapabilityFPGAMemoryAccessesALTERA = 5898, CapabilityFPGAMemoryAccessesINTEL = 5898, + CapabilityFPGAClusterAttributesALTERA = 5904, CapabilityFPGAClusterAttributesINTEL = 5904, + CapabilityLoopFuseALTERA = 5906, CapabilityLoopFuseINTEL = 5906, + CapabilityFPGADSPControlALTERA = 5908, + CapabilityFPGADSPControlINTEL = 5908, CapabilityMemoryAccessAliasingINTEL = 5910, + CapabilityFPGAInvocationPipeliningAttributesALTERA = 5916, + CapabilityFPGAInvocationPipeliningAttributesINTEL = 5916, + CapabilityFPGABufferLocationALTERA = 5920, CapabilityFPGABufferLocationINTEL = 5920, + CapabilityArbitraryPrecisionFixedPointALTERA = 5922, CapabilityArbitraryPrecisionFixedPointINTEL = 5922, + CapabilityUSMStorageClassesALTERA = 5935, CapabilityUSMStorageClassesINTEL = 5935, + CapabilityRuntimeAlignedAttributeALTERA = 5939, + CapabilityRuntimeAlignedAttributeINTEL = 5939, + CapabilityIOPipesALTERA = 5943, CapabilityIOPipesINTEL = 5943, + CapabilityBlockingPipesALTERA = 5945, CapabilityBlockingPipesINTEL = 5945, + CapabilityFPGARegALTERA = 5948, CapabilityFPGARegINTEL = 5948, CapabilityDotProductInputAll = 6016, CapabilityDotProductInputAllKHR = 6016, @@ -1101,16 +1354,54 @@ enum Capability { CapabilityDotProduct = 6019, CapabilityDotProductKHR = 6019, CapabilityRayCullMaskKHR = 6020, + CapabilityCooperativeMatrixKHR = 6022, + CapabilityReplicatedCompositesEXT = 6024, CapabilityBitInstructions = 6025, CapabilityGroupNonUniformRotateKHR = 6026, + CapabilityFloatControls2 = 6029, + CapabilityFMAKHR = 6030, CapabilityAtomicFloat32AddEXT = 6033, CapabilityAtomicFloat64AddEXT = 6034, - CapabilityLongConstantCompositeINTEL = 6089, + CapabilityLongCompositesINTEL = 6089, + CapabilityOptNoneEXT = 6094, CapabilityOptNoneINTEL = 6094, CapabilityAtomicFloat16AddEXT = 6095, CapabilityDebugInfoModuleINTEL = 6114, + CapabilityBFloat16ConversionINTEL = 6115, CapabilitySplitBarrierINTEL = 6141, + CapabilityArithmeticFenceEXT = 6144, + CapabilityFPGAClusterAttributesV2ALTERA = 6150, + CapabilityFPGAClusterAttributesV2INTEL = 6150, + CapabilityFPGAKernelAttributesv2INTEL = 6161, + CapabilityTaskSequenceALTERA = 6162, + CapabilityTaskSequenceINTEL = 6162, + CapabilityFPMaxErrorINTEL = 6169, + CapabilityFPGALatencyControlALTERA = 6171, + CapabilityFPGALatencyControlINTEL = 6171, + CapabilityFPGAArgumentInterfacesALTERA = 6174, + CapabilityFPGAArgumentInterfacesINTEL = 6174, + CapabilityGlobalVariableHostAccessINTEL = 6187, + CapabilityGlobalVariableFPGADecorationsALTERA = 6189, + CapabilityGlobalVariableFPGADecorationsINTEL = 6189, + CapabilitySubgroupBufferPrefetchINTEL = 6220, + CapabilitySubgroup2DBlockIOINTEL = 6228, + CapabilitySubgroup2DBlockTransformINTEL = 6229, + CapabilitySubgroup2DBlockTransposeINTEL = 6230, + CapabilitySubgroupMatrixMultiplyAccumulateINTEL = 6236, + CapabilityTernaryBitwiseFunctionINTEL = 6241, + CapabilityUntypedVariableLengthArrayINTEL = 6243, + CapabilitySpecConditionalINTEL = 6245, + CapabilityFunctionVariantsINTEL = 6246, CapabilityGroupUniformArithmeticKHR = 6400, + CapabilityTensorFloat32RoundingINTEL = 6425, + CapabilityMaskedGatherScatterINTEL = 6427, + CapabilityCacheControlsINTEL = 6441, + CapabilityRegisterLimitsINTEL = 6460, + CapabilityBindlessImagesINTEL = 6528, + CapabilityDotProductFloat16AccFloat32VALVE = 6912, + CapabilityDotProductFloat16AccFloat16VALVE = 6913, + CapabilityDotProductBFloat16AccVALVE = 6914, + CapabilityDotProductFloat8AccFloat32VALVE = 6915, CapabilityMax = 0x7fffffff, }; @@ -1123,8 +1414,10 @@ enum RayFlagsShift { RayFlagsCullFrontFacingTrianglesKHRShift = 5, RayFlagsCullOpaqueKHRShift = 6, RayFlagsCullNoOpaqueKHRShift = 7, + RayFlagsSkipBuiltinPrimitivesNVShift = 8, RayFlagsSkipTrianglesKHRShift = 8, RayFlagsSkipAABBsKHRShift = 9, + RayFlagsForceOpacityMicromap2StateEXTShift = 10, RayFlagsMax = 0x7fffffff, }; @@ -1138,8 +1431,10 @@ enum RayFlagsMask { RayFlagsCullFrontFacingTrianglesKHRMask = 0x00000020, RayFlagsCullOpaqueKHRMask = 0x00000040, RayFlagsCullNoOpaqueKHRMask = 0x00000080, + RayFlagsSkipBuiltinPrimitivesNVMask = 0x00000100, RayFlagsSkipTrianglesKHRMask = 0x00000100, RayFlagsSkipAABBsKHRMask = 0x00000200, + RayFlagsForceOpacityMicromap2StateEXTMask = 0x00000400, }; enum RayQueryIntersection { @@ -1215,6 +1510,212 @@ enum PackedVectorFormat { PackedVectorFormatMax = 0x7fffffff, }; +enum CooperativeMatrixOperandsShift { + CooperativeMatrixOperandsMatrixASignedComponentsKHRShift = 0, + CooperativeMatrixOperandsMatrixBSignedComponentsKHRShift = 1, + CooperativeMatrixOperandsMatrixCSignedComponentsKHRShift = 2, + CooperativeMatrixOperandsMatrixResultSignedComponentsKHRShift = 3, + CooperativeMatrixOperandsSaturatingAccumulationKHRShift = 4, + CooperativeMatrixOperandsMax = 0x7fffffff, +}; + +enum CooperativeMatrixOperandsMask { + CooperativeMatrixOperandsMaskNone = 0, + CooperativeMatrixOperandsMatrixASignedComponentsKHRMask = 0x00000001, + CooperativeMatrixOperandsMatrixBSignedComponentsKHRMask = 0x00000002, + CooperativeMatrixOperandsMatrixCSignedComponentsKHRMask = 0x00000004, + CooperativeMatrixOperandsMatrixResultSignedComponentsKHRMask = 0x00000008, + CooperativeMatrixOperandsSaturatingAccumulationKHRMask = 0x00000010, +}; + +enum CooperativeMatrixLayout { + CooperativeMatrixLayoutRowMajorKHR = 0, + CooperativeMatrixLayoutColumnMajorKHR = 1, + CooperativeMatrixLayoutRowBlockedInterleavedARM = 4202, + CooperativeMatrixLayoutColumnBlockedInterleavedARM = 4203, + CooperativeMatrixLayoutMax = 0x7fffffff, +}; + +enum CooperativeMatrixUse { + CooperativeMatrixUseMatrixAKHR = 0, + CooperativeMatrixUseMatrixBKHR = 1, + CooperativeMatrixUseMatrixAccumulatorKHR = 2, + CooperativeMatrixUseMax = 0x7fffffff, +}; + +enum CooperativeMatrixReduceShift { + CooperativeMatrixReduceRowShift = 0, + CooperativeMatrixReduceColumnShift = 1, + CooperativeMatrixReduce2x2Shift = 2, + CooperativeMatrixReduceMax = 0x7fffffff, +}; + +enum CooperativeMatrixReduceMask { + CooperativeMatrixReduceMaskNone = 0, + CooperativeMatrixReduceRowMask = 0x00000001, + CooperativeMatrixReduceColumnMask = 0x00000002, + CooperativeMatrixReduce2x2Mask = 0x00000004, +}; + +enum TensorClampMode { + TensorClampModeUndefined = 0, + TensorClampModeConstant = 1, + TensorClampModeClampToEdge = 2, + TensorClampModeRepeat = 3, + TensorClampModeRepeatMirrored = 4, + TensorClampModeMax = 0x7fffffff, +}; + +enum TensorAddressingOperandsShift { + TensorAddressingOperandsTensorViewShift = 0, + TensorAddressingOperandsDecodeFuncShift = 1, + TensorAddressingOperandsMax = 0x7fffffff, +}; + +enum TensorAddressingOperandsMask { + TensorAddressingOperandsMaskNone = 0, + TensorAddressingOperandsTensorViewMask = 0x00000001, + TensorAddressingOperandsDecodeFuncMask = 0x00000002, +}; + +enum TensorOperandsShift { + TensorOperandsNontemporalARMShift = 0, + TensorOperandsOutOfBoundsValueARMShift = 1, + TensorOperandsMakeElementAvailableARMShift = 2, + TensorOperandsMakeElementVisibleARMShift = 3, + TensorOperandsNonPrivateElementARMShift = 4, + TensorOperandsMax = 0x7fffffff, +}; + +enum TensorOperandsMask { + TensorOperandsMaskNone = 0, + TensorOperandsNontemporalARMMask = 0x00000001, + TensorOperandsOutOfBoundsValueARMMask = 0x00000002, + TensorOperandsMakeElementAvailableARMMask = 0x00000004, + TensorOperandsMakeElementVisibleARMMask = 0x00000008, + TensorOperandsNonPrivateElementARMMask = 0x00000010, +}; + +enum InitializationModeQualifier { + InitializationModeQualifierInitOnDeviceReprogramALTERA = 0, + InitializationModeQualifierInitOnDeviceReprogramINTEL = 0, + InitializationModeQualifierInitOnDeviceResetALTERA = 1, + InitializationModeQualifierInitOnDeviceResetINTEL = 1, + InitializationModeQualifierMax = 0x7fffffff, +}; + +enum HostAccessQualifier { + HostAccessQualifierNoneINTEL = 0, + HostAccessQualifierReadINTEL = 1, + HostAccessQualifierWriteINTEL = 2, + HostAccessQualifierReadWriteINTEL = 3, + HostAccessQualifierMax = 0x7fffffff, +}; + +enum LoadCacheControl { + LoadCacheControlUncachedINTEL = 0, + LoadCacheControlCachedINTEL = 1, + LoadCacheControlStreamingINTEL = 2, + LoadCacheControlInvalidateAfterReadINTEL = 3, + LoadCacheControlConstCachedINTEL = 4, + LoadCacheControlMax = 0x7fffffff, +}; + +enum StoreCacheControl { + StoreCacheControlUncachedINTEL = 0, + StoreCacheControlWriteThroughINTEL = 1, + StoreCacheControlWriteBackINTEL = 2, + StoreCacheControlStreamingINTEL = 3, + StoreCacheControlMax = 0x7fffffff, +}; + +enum NamedMaximumNumberOfRegisters { + NamedMaximumNumberOfRegistersAutoINTEL = 0, + NamedMaximumNumberOfRegistersMax = 0x7fffffff, +}; + +enum MatrixMultiplyAccumulateOperandsShift { + MatrixMultiplyAccumulateOperandsMatrixASignedComponentsINTELShift = 0, + MatrixMultiplyAccumulateOperandsMatrixBSignedComponentsINTELShift = 1, + MatrixMultiplyAccumulateOperandsMatrixCBFloat16INTELShift = 2, + MatrixMultiplyAccumulateOperandsMatrixResultBFloat16INTELShift = 3, + MatrixMultiplyAccumulateOperandsMatrixAPackedInt8INTELShift = 4, + MatrixMultiplyAccumulateOperandsMatrixBPackedInt8INTELShift = 5, + MatrixMultiplyAccumulateOperandsMatrixAPackedInt4INTELShift = 6, + MatrixMultiplyAccumulateOperandsMatrixBPackedInt4INTELShift = 7, + MatrixMultiplyAccumulateOperandsMatrixATF32INTELShift = 8, + MatrixMultiplyAccumulateOperandsMatrixBTF32INTELShift = 9, + MatrixMultiplyAccumulateOperandsMatrixAPackedFloat16INTELShift = 10, + MatrixMultiplyAccumulateOperandsMatrixBPackedFloat16INTELShift = 11, + MatrixMultiplyAccumulateOperandsMatrixAPackedBFloat16INTELShift = 12, + MatrixMultiplyAccumulateOperandsMatrixBPackedBFloat16INTELShift = 13, + MatrixMultiplyAccumulateOperandsMax = 0x7fffffff, +}; + +enum MatrixMultiplyAccumulateOperandsMask { + MatrixMultiplyAccumulateOperandsMaskNone = 0, + MatrixMultiplyAccumulateOperandsMatrixASignedComponentsINTELMask = 0x00000001, + MatrixMultiplyAccumulateOperandsMatrixBSignedComponentsINTELMask = 0x00000002, + MatrixMultiplyAccumulateOperandsMatrixCBFloat16INTELMask = 0x00000004, + MatrixMultiplyAccumulateOperandsMatrixResultBFloat16INTELMask = 0x00000008, + MatrixMultiplyAccumulateOperandsMatrixAPackedInt8INTELMask = 0x00000010, + MatrixMultiplyAccumulateOperandsMatrixBPackedInt8INTELMask = 0x00000020, + MatrixMultiplyAccumulateOperandsMatrixAPackedInt4INTELMask = 0x00000040, + MatrixMultiplyAccumulateOperandsMatrixBPackedInt4INTELMask = 0x00000080, + MatrixMultiplyAccumulateOperandsMatrixATF32INTELMask = 0x00000100, + MatrixMultiplyAccumulateOperandsMatrixBTF32INTELMask = 0x00000200, + MatrixMultiplyAccumulateOperandsMatrixAPackedFloat16INTELMask = 0x00000400, + MatrixMultiplyAccumulateOperandsMatrixBPackedFloat16INTELMask = 0x00000800, + MatrixMultiplyAccumulateOperandsMatrixAPackedBFloat16INTELMask = 0x00001000, + MatrixMultiplyAccumulateOperandsMatrixBPackedBFloat16INTELMask = 0x00002000, +}; + +enum RawAccessChainOperandsShift { + RawAccessChainOperandsRobustnessPerComponentNVShift = 0, + RawAccessChainOperandsRobustnessPerElementNVShift = 1, + RawAccessChainOperandsMax = 0x7fffffff, +}; + +enum RawAccessChainOperandsMask { + RawAccessChainOperandsMaskNone = 0, + RawAccessChainOperandsRobustnessPerComponentNVMask = 0x00000001, + RawAccessChainOperandsRobustnessPerElementNVMask = 0x00000002, +}; + +enum FPEncoding { + FPEncodingBFloat16KHR = 0, + FPEncodingFloat8E4M3EXT = 4214, + FPEncodingFloat8E5M2EXT = 4215, + FPEncodingMax = 0x7fffffff, +}; + +enum CooperativeVectorMatrixLayout { + CooperativeVectorMatrixLayoutRowMajorNV = 0, + CooperativeVectorMatrixLayoutColumnMajorNV = 1, + CooperativeVectorMatrixLayoutInferencingOptimalNV = 2, + CooperativeVectorMatrixLayoutTrainingOptimalNV = 3, + CooperativeVectorMatrixLayoutMax = 0x7fffffff, +}; + +enum ComponentType { + ComponentTypeFloat16NV = 0, + ComponentTypeFloat32NV = 1, + ComponentTypeFloat64NV = 2, + ComponentTypeSignedInt8NV = 3, + ComponentTypeSignedInt16NV = 4, + ComponentTypeSignedInt32NV = 5, + ComponentTypeSignedInt64NV = 6, + ComponentTypeUnsignedInt8NV = 7, + ComponentTypeUnsignedInt16NV = 8, + ComponentTypeUnsignedInt32NV = 9, + ComponentTypeUnsignedInt64NV = 10, + ComponentTypeSignedInt8PackedNV = 1000491000, + ComponentTypeUnsignedInt8PackedNV = 1000491001, + ComponentTypeFloatE4M3NV = 1000491002, + ComponentTypeFloatE5M2NV = 1000491003, + ComponentTypeMax = 0x7fffffff, +}; + enum Op { OpNop = 0, OpUndef = 1, @@ -1560,14 +2061,39 @@ enum Op { OpPtrEqual = 401, OpPtrNotEqual = 402, OpPtrDiff = 403, + OpColorAttachmentReadEXT = 4160, + OpDepthAttachmentReadEXT = 4161, + OpStencilAttachmentReadEXT = 4162, + OpTypeTensorARM = 4163, + OpTensorReadARM = 4164, + OpTensorWriteARM = 4165, + OpTensorQuerySizeARM = 4166, + OpGraphConstantARM = 4181, + OpGraphEntryPointARM = 4182, + OpGraphARM = 4183, + OpGraphInputARM = 4184, + OpGraphSetOutputARM = 4185, + OpGraphEndARM = 4186, + OpTypeGraphARM = 4190, OpTerminateInvocation = 4416, + OpTypeUntypedPointerKHR = 4417, + OpUntypedVariableKHR = 4418, + OpUntypedAccessChainKHR = 4419, + OpUntypedInBoundsAccessChainKHR = 4420, OpSubgroupBallotKHR = 4421, OpSubgroupFirstInvocationKHR = 4422, + OpUntypedPtrAccessChainKHR = 4423, + OpUntypedInBoundsPtrAccessChainKHR = 4424, + OpUntypedArrayLengthKHR = 4425, + OpUntypedPrefetchKHR = 4426, + OpFmaKHR = 4427, OpSubgroupAllKHR = 4428, OpSubgroupAnyKHR = 4429, OpSubgroupAllEqualKHR = 4430, OpGroupNonUniformRotateKHR = 4431, OpSubgroupReadInvocationKHR = 4432, + OpExtInstWithForwardRefsKHR = 4433, + OpUntypedGroupAsyncCopyKHR = 4434, OpTraceRayKHR = 4445, OpExecuteCallableKHR = 4446, OpConvertUToAccelerationStructureKHR = 4447, @@ -1585,6 +2111,14 @@ enum Op { OpUDotAccSatKHR = 4454, OpSUDotAccSat = 4455, OpSUDotAccSatKHR = 4455, + OpTypeCooperativeMatrixKHR = 4456, + OpCooperativeMatrixLoadKHR = 4457, + OpCooperativeMatrixStoreKHR = 4458, + OpCooperativeMatrixMulAddKHR = 4459, + OpCooperativeMatrixLengthKHR = 4460, + OpConstantCompositeReplicateEXT = 4461, + OpSpecConstantCompositeReplicateEXT = 4462, + OpCompositeConstructReplicateEXT = 4463, OpTypeRayQueryKHR = 4472, OpRayQueryInitializeKHR = 4473, OpRayQueryTerminateKHR = 4474, @@ -1592,6 +2126,18 @@ enum Op { OpRayQueryConfirmIntersectionKHR = 4476, OpRayQueryProceedKHR = 4477, OpRayQueryGetIntersectionTypeKHR = 4479, + OpImageSampleWeightedQCOM = 4480, + OpImageBoxFilterQCOM = 4481, + OpImageBlockMatchSSDQCOM = 4482, + OpImageBlockMatchSADQCOM = 4483, + OpBitCastArrayQCOM = 4497, + OpImageBlockMatchWindowSSDQCOM = 4500, + OpImageBlockMatchWindowSADQCOM = 4501, + OpImageBlockMatchGatherSSDQCOM = 4502, + OpImageBlockMatchGatherSADQCOM = 4503, + OpCompositeConstructCoopMatQCOM = 4540, + OpCompositeExtractCoopMatQCOM = 4541, + OpExtractSubArrayQCOM = 4542, OpGroupIAddNonUniformAMD = 5000, OpGroupFAddNonUniformAMD = 5001, OpGroupFMinNonUniformAMD = 5002, @@ -1603,11 +2149,106 @@ enum Op { OpFragmentMaskFetchAMD = 5011, OpFragmentFetchAMD = 5012, OpReadClockKHR = 5056, + OpAllocateNodePayloadsAMDX = 5074, + OpEnqueueNodePayloadsAMDX = 5075, + OpTypeNodePayloadArrayAMDX = 5076, + OpFinishWritingNodePayloadAMDX = 5078, + OpNodePayloadArrayLengthAMDX = 5090, + OpIsNodePayloadValidAMDX = 5101, + OpConstantStringAMDX = 5103, + OpSpecConstantStringAMDX = 5104, + OpGroupNonUniformQuadAllKHR = 5110, + OpGroupNonUniformQuadAnyKHR = 5111, + OpTypeBufferEXT = 5115, + OpBufferPointerEXT = 5119, + OpAbortKHR = 5121, + OpUntypedImageTexelPointerEXT = 5126, + OpMemberDecorateIdEXT = 5127, + OpConstantSizeOfEXT = 5129, + OpConstantDataKHR = 5147, + OpSpecConstantDataKHR = 5148, + OpPoisonKHR = 5158, + OpFreezeKHR = 5159, + OpHitObjectRecordHitMotionNV = 5249, + OpHitObjectRecordHitWithIndexMotionNV = 5250, + OpHitObjectRecordMissMotionNV = 5251, + OpHitObjectGetWorldToObjectNV = 5252, + OpHitObjectGetObjectToWorldNV = 5253, + OpHitObjectGetObjectRayDirectionNV = 5254, + OpHitObjectGetObjectRayOriginNV = 5255, + OpHitObjectTraceRayMotionNV = 5256, + OpHitObjectGetShaderRecordBufferHandleNV = 5257, + OpHitObjectGetShaderBindingTableRecordIndexNV = 5258, + OpHitObjectRecordEmptyNV = 5259, + OpHitObjectTraceRayNV = 5260, + OpHitObjectRecordHitNV = 5261, + OpHitObjectRecordHitWithIndexNV = 5262, + OpHitObjectRecordMissNV = 5263, + OpHitObjectExecuteShaderNV = 5264, + OpHitObjectGetCurrentTimeNV = 5265, + OpHitObjectGetAttributesNV = 5266, + OpHitObjectGetHitKindNV = 5267, + OpHitObjectGetPrimitiveIndexNV = 5268, + OpHitObjectGetGeometryIndexNV = 5269, + OpHitObjectGetInstanceIdNV = 5270, + OpHitObjectGetInstanceCustomIndexNV = 5271, + OpHitObjectGetWorldRayDirectionNV = 5272, + OpHitObjectGetWorldRayOriginNV = 5273, + OpHitObjectGetRayTMaxNV = 5274, + OpHitObjectGetRayTMinNV = 5275, + OpHitObjectIsEmptyNV = 5276, + OpHitObjectIsHitNV = 5277, + OpHitObjectIsMissNV = 5278, + OpReorderThreadWithHitObjectNV = 5279, + OpReorderThreadWithHintNV = 5280, + OpTypeHitObjectNV = 5281, OpImageSampleFootprintNV = 5283, + OpTypeCooperativeVectorNV = 5288, + OpTypeVectorIdEXT = 5288, + OpCooperativeVectorMatrixMulNV = 5289, + OpCooperativeVectorOuterProductAccumulateNV = 5290, + OpCooperativeVectorReduceSumAccumulateNV = 5291, + OpCooperativeVectorMatrixMulAddNV = 5292, + OpCooperativeMatrixConvertNV = 5293, OpEmitMeshTasksEXT = 5294, OpSetMeshOutputsEXT = 5295, + OpGroupNonUniformPartitionEXT = 5296, OpGroupNonUniformPartitionNV = 5296, OpWritePackedPrimitiveIndices4x8NV = 5299, + OpFetchMicroTriangleVertexPositionNV = 5300, + OpFetchMicroTriangleVertexBarycentricNV = 5301, + OpCooperativeVectorLoadNV = 5302, + OpCooperativeVectorStoreNV = 5303, + OpHitObjectRecordFromQueryEXT = 5304, + OpHitObjectRecordMissEXT = 5305, + OpHitObjectRecordMissMotionEXT = 5306, + OpHitObjectGetIntersectionTriangleVertexPositionsEXT = 5307, + OpHitObjectGetRayFlagsEXT = 5308, + OpHitObjectSetShaderBindingTableRecordIndexEXT = 5309, + OpHitObjectReorderExecuteShaderEXT = 5310, + OpHitObjectTraceReorderExecuteEXT = 5311, + OpHitObjectTraceMotionReorderExecuteEXT = 5312, + OpTypeHitObjectEXT = 5313, + OpReorderThreadWithHintEXT = 5314, + OpReorderThreadWithHitObjectEXT = 5315, + OpHitObjectTraceRayEXT = 5316, + OpHitObjectTraceRayMotionEXT = 5317, + OpHitObjectRecordEmptyEXT = 5318, + OpHitObjectExecuteShaderEXT = 5319, + OpHitObjectGetCurrentTimeEXT = 5320, + OpHitObjectGetAttributesEXT = 5321, + OpHitObjectGetHitKindEXT = 5322, + OpHitObjectGetPrimitiveIndexEXT = 5323, + OpHitObjectGetGeometryIndexEXT = 5324, + OpHitObjectGetInstanceIdEXT = 5325, + OpHitObjectGetInstanceCustomIndexEXT = 5326, + OpHitObjectGetObjectRayOriginEXT = 5327, + OpHitObjectGetObjectRayDirectionEXT = 5328, + OpHitObjectGetWorldRayDirectionEXT = 5329, + OpHitObjectGetWorldRayOriginEXT = 5330, + OpHitObjectGetObjectToWorldEXT = 5331, + OpHitObjectGetWorldToObjectEXT = 5332, + OpHitObjectGetRayTMaxEXT = 5333, OpReportIntersectionKHR = 5334, OpReportIntersectionNV = 5334, OpIgnoreIntersectionNV = 5335, @@ -1615,9 +2256,19 @@ enum Op { OpTraceNV = 5337, OpTraceMotionNV = 5338, OpTraceRayMotionNV = 5339, + OpRayQueryGetIntersectionTriangleVertexPositionsKHR = 5340, OpTypeAccelerationStructureKHR = 5341, OpTypeAccelerationStructureNV = 5341, OpExecuteCallableNV = 5344, + OpRayQueryGetClusterIdNV = 5345, + OpRayQueryGetIntersectionClusterIdNV = 5345, + OpHitObjectGetClusterIdNV = 5346, + OpHitObjectGetRayTMinEXT = 5347, + OpHitObjectGetShaderBindingTableRecordIndexEXT = 5348, + OpHitObjectGetShaderRecordBufferHandleEXT = 5349, + OpHitObjectIsEmptyEXT = 5350, + OpHitObjectIsHitEXT = 5351, + OpHitObjectIsMissEXT = 5352, OpTypeCooperativeMatrixNV = 5358, OpCooperativeMatrixLoadNV = 5359, OpCooperativeMatrixStoreNV = 5360, @@ -1625,9 +2276,26 @@ enum Op { OpCooperativeMatrixLengthNV = 5362, OpBeginInvocationInterlockEXT = 5364, OpEndInvocationInterlockEXT = 5365, + OpCooperativeMatrixReduceNV = 5366, + OpCooperativeMatrixLoadTensorNV = 5367, + OpCooperativeMatrixStoreTensorNV = 5368, + OpCooperativeMatrixPerElementOpNV = 5369, + OpTypeTensorLayoutNV = 5370, + OpTypeTensorViewNV = 5371, + OpCreateTensorLayoutNV = 5372, + OpTensorLayoutSetDimensionNV = 5373, + OpTensorLayoutSetStrideNV = 5374, + OpTensorLayoutSliceNV = 5375, + OpTensorLayoutSetClampValueNV = 5376, + OpCreateTensorViewNV = 5377, + OpTensorViewSetDimensionNV = 5378, + OpTensorViewSetStrideNV = 5379, OpDemoteToHelperInvocation = 5380, OpDemoteToHelperInvocationEXT = 5380, OpIsHelperInvocationEXT = 5381, + OpTensorViewSetClipNV = 5382, + OpTensorLayoutSetBlockSizeNV = 5384, + OpCooperativeMatrixTransposeNV = 5390, OpConvertUToImageNV = 5391, OpConvertUToSamplerNV = 5392, OpConvertImageToUNV = 5393, @@ -1635,6 +2303,20 @@ enum Op { OpConvertUToSampledImageNV = 5395, OpConvertSampledImageToUNV = 5396, OpSamplerImageAddressingModeNV = 5397, + OpRawAccessChainNV = 5398, + OpRayQueryGetIntersectionSpherePositionNV = 5427, + OpRayQueryGetIntersectionSphereRadiusNV = 5428, + OpRayQueryGetIntersectionLSSPositionsNV = 5429, + OpRayQueryGetIntersectionLSSRadiiNV = 5430, + OpRayQueryGetIntersectionLSSHitValueNV = 5431, + OpHitObjectGetSpherePositionNV = 5432, + OpHitObjectGetSphereRadiusNV = 5433, + OpHitObjectGetLSSPositionsNV = 5434, + OpHitObjectGetLSSRadiiNV = 5435, + OpHitObjectIsSphereHitNV = 5436, + OpHitObjectIsLSSHitNV = 5437, + OpRayQueryIsSphereHitNV = 5438, + OpRayQueryIsLSSHitNV = 5439, OpSubgroupShuffleINTEL = 5571, OpSubgroupShuffleDownINTEL = 5572, OpSubgroupShuffleUpINTEL = 5573, @@ -1793,23 +2475,41 @@ enum Op { OpVariableLengthArrayINTEL = 5818, OpSaveMemoryINTEL = 5819, OpRestoreMemoryINTEL = 5820, + OpArbitraryFloatSinCosPiALTERA = 5840, OpArbitraryFloatSinCosPiINTEL = 5840, + OpArbitraryFloatCastALTERA = 5841, OpArbitraryFloatCastINTEL = 5841, + OpArbitraryFloatCastFromIntALTERA = 5842, OpArbitraryFloatCastFromIntINTEL = 5842, + OpArbitraryFloatCastToIntALTERA = 5843, OpArbitraryFloatCastToIntINTEL = 5843, + OpArbitraryFloatAddALTERA = 5846, OpArbitraryFloatAddINTEL = 5846, + OpArbitraryFloatSubALTERA = 5847, OpArbitraryFloatSubINTEL = 5847, + OpArbitraryFloatMulALTERA = 5848, OpArbitraryFloatMulINTEL = 5848, + OpArbitraryFloatDivALTERA = 5849, OpArbitraryFloatDivINTEL = 5849, + OpArbitraryFloatGTALTERA = 5850, OpArbitraryFloatGTINTEL = 5850, + OpArbitraryFloatGEALTERA = 5851, OpArbitraryFloatGEINTEL = 5851, + OpArbitraryFloatLTALTERA = 5852, OpArbitraryFloatLTINTEL = 5852, + OpArbitraryFloatLEALTERA = 5853, OpArbitraryFloatLEINTEL = 5853, + OpArbitraryFloatEQALTERA = 5854, OpArbitraryFloatEQINTEL = 5854, + OpArbitraryFloatRecipALTERA = 5855, OpArbitraryFloatRecipINTEL = 5855, + OpArbitraryFloatRSqrtALTERA = 5856, OpArbitraryFloatRSqrtINTEL = 5856, + OpArbitraryFloatCbrtALTERA = 5857, OpArbitraryFloatCbrtINTEL = 5857, + OpArbitraryFloatHypotALTERA = 5858, OpArbitraryFloatHypotINTEL = 5858, + OpArbitraryFloatSqrtALTERA = 5859, OpArbitraryFloatSqrtINTEL = 5859, OpArbitraryFloatLogINTEL = 5860, OpArbitraryFloatLog2INTEL = 5861, @@ -1838,21 +2538,37 @@ enum Op { OpAliasDomainDeclINTEL = 5911, OpAliasScopeDeclINTEL = 5912, OpAliasScopeListDeclINTEL = 5913, + OpFixedSqrtALTERA = 5923, OpFixedSqrtINTEL = 5923, + OpFixedRecipALTERA = 5924, OpFixedRecipINTEL = 5924, + OpFixedRsqrtALTERA = 5925, OpFixedRsqrtINTEL = 5925, + OpFixedSinALTERA = 5926, OpFixedSinINTEL = 5926, + OpFixedCosALTERA = 5927, OpFixedCosINTEL = 5927, + OpFixedSinCosALTERA = 5928, OpFixedSinCosINTEL = 5928, + OpFixedSinPiALTERA = 5929, OpFixedSinPiINTEL = 5929, + OpFixedCosPiALTERA = 5930, OpFixedCosPiINTEL = 5930, + OpFixedSinCosPiALTERA = 5931, OpFixedSinCosPiINTEL = 5931, + OpFixedLogALTERA = 5932, OpFixedLogINTEL = 5932, + OpFixedExpALTERA = 5933, OpFixedExpINTEL = 5933, + OpPtrCastToCrossWorkgroupALTERA = 5934, OpPtrCastToCrossWorkgroupINTEL = 5934, + OpCrossWorkgroupCastToPtrALTERA = 5938, OpCrossWorkgroupCastToPtrINTEL = 5938, + OpReadPipeBlockingALTERA = 5946, OpReadPipeBlockingINTEL = 5946, + OpWritePipeBlockingALTERA = 5947, OpWritePipeBlockingINTEL = 5947, + OpFPGARegALTERA = 5949, OpFPGARegINTEL = 5949, OpRayQueryGetRayTMinKHR = 6016, OpRayQueryGetRayFlagsKHR = 6017, @@ -1876,8 +2592,38 @@ enum Op { OpTypeStructContinuedINTEL = 6090, OpConstantCompositeContinuedINTEL = 6091, OpSpecConstantCompositeContinuedINTEL = 6092, + OpCompositeConstructContinuedINTEL = 6096, + OpConvertFToBF16INTEL = 6116, + OpConvertBF16ToFINTEL = 6117, OpControlBarrierArriveINTEL = 6142, OpControlBarrierWaitINTEL = 6143, + OpArithmeticFenceEXT = 6145, + OpTaskSequenceCreateALTERA = 6163, + OpTaskSequenceCreateINTEL = 6163, + OpTaskSequenceAsyncALTERA = 6164, + OpTaskSequenceAsyncINTEL = 6164, + OpTaskSequenceGetALTERA = 6165, + OpTaskSequenceGetINTEL = 6165, + OpTaskSequenceReleaseALTERA = 6166, + OpTaskSequenceReleaseINTEL = 6166, + OpTypeTaskSequenceALTERA = 6199, + OpTypeTaskSequenceINTEL = 6199, + OpSubgroupBlockPrefetchINTEL = 6221, + OpSubgroup2DBlockLoadINTEL = 6231, + OpSubgroup2DBlockLoadTransformINTEL = 6232, + OpSubgroup2DBlockLoadTransposeINTEL = 6233, + OpSubgroup2DBlockPrefetchINTEL = 6234, + OpSubgroup2DBlockStoreINTEL = 6235, + OpSubgroupMatrixMultiplyAccumulateINTEL = 6237, + OpBitwiseFunctionINTEL = 6242, + OpUntypedVariableLengthArrayINTEL = 6244, + OpConditionalExtensionINTEL = 6248, + OpConditionalEntryPointINTEL = 6249, + OpConditionalCapabilityINTEL = 6250, + OpSpecConstantTargetINTEL = 6251, + OpSpecConstantArchitectureINTEL = 6252, + OpSpecConstantCapabilitiesINTEL = 6253, + OpConditionalCopyObjectINTEL = 6254, OpGroupIMulKHR = 6401, OpGroupFMulKHR = 6402, OpGroupBitwiseAndKHR = 6403, @@ -1886,6 +2632,15 @@ enum Op { OpGroupLogicalAndKHR = 6406, OpGroupLogicalOrKHR = 6407, OpGroupLogicalXorKHR = 6408, + OpRoundFToTF32INTEL = 6426, + OpMaskedGatherINTEL = 6428, + OpMaskedScatterINTEL = 6429, + OpConvertHandleToImageINTEL = 6529, + OpConvertHandleToSamplerINTEL = 6530, + OpConvertHandleToSampledImageINTEL = 6531, + OpFDot2MixAcc32VALVE = 6916, + OpFDot2MixAcc16VALVE = 6917, + OpFDot4MixAcc32VALVE = 6918, OpMax = 0x7fffffff, }; @@ -2241,14 +2996,39 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpPtrEqual: *hasResult = true; *hasResultType = true; break; case OpPtrNotEqual: *hasResult = true; *hasResultType = true; break; case OpPtrDiff: *hasResult = true; *hasResultType = true; break; + case OpColorAttachmentReadEXT: *hasResult = true; *hasResultType = true; break; + case OpDepthAttachmentReadEXT: *hasResult = true; *hasResultType = true; break; + case OpStencilAttachmentReadEXT: *hasResult = true; *hasResultType = true; break; + case OpTypeTensorARM: *hasResult = true; *hasResultType = false; break; + case OpTensorReadARM: *hasResult = true; *hasResultType = true; break; + case OpTensorWriteARM: *hasResult = false; *hasResultType = false; break; + case OpTensorQuerySizeARM: *hasResult = true; *hasResultType = true; break; + case OpGraphConstantARM: *hasResult = true; *hasResultType = true; break; + case OpGraphEntryPointARM: *hasResult = false; *hasResultType = false; break; + case OpGraphARM: *hasResult = true; *hasResultType = true; break; + case OpGraphInputARM: *hasResult = true; *hasResultType = true; break; + case OpGraphSetOutputARM: *hasResult = false; *hasResultType = false; break; + case OpGraphEndARM: *hasResult = false; *hasResultType = false; break; + case OpTypeGraphARM: *hasResult = true; *hasResultType = false; break; case OpTerminateInvocation: *hasResult = false; *hasResultType = false; break; + case OpTypeUntypedPointerKHR: *hasResult = true; *hasResultType = false; break; + case OpUntypedVariableKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedAccessChainKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedInBoundsAccessChainKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupBallotKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupFirstInvocationKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedPtrAccessChainKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedInBoundsPtrAccessChainKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedArrayLengthKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedPrefetchKHR: *hasResult = false; *hasResultType = false; break; + case OpFmaKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupAllKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupAnyKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupAllEqualKHR: *hasResult = true; *hasResultType = true; break; case OpGroupNonUniformRotateKHR: *hasResult = true; *hasResultType = true; break; case OpSubgroupReadInvocationKHR: *hasResult = true; *hasResultType = true; break; + case OpExtInstWithForwardRefsKHR: *hasResult = true; *hasResultType = true; break; + case OpUntypedGroupAsyncCopyKHR: *hasResult = true; *hasResultType = true; break; case OpTraceRayKHR: *hasResult = false; *hasResultType = false; break; case OpExecuteCallableKHR: *hasResult = false; *hasResultType = false; break; case OpConvertUToAccelerationStructureKHR: *hasResult = true; *hasResultType = true; break; @@ -2260,6 +3040,14 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpSDotAccSat: *hasResult = true; *hasResultType = true; break; case OpUDotAccSat: *hasResult = true; *hasResultType = true; break; case OpSUDotAccSat: *hasResult = true; *hasResultType = true; break; + case OpTypeCooperativeMatrixKHR: *hasResult = true; *hasResultType = false; break; + case OpCooperativeMatrixLoadKHR: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixStoreKHR: *hasResult = false; *hasResultType = false; break; + case OpCooperativeMatrixMulAddKHR: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixLengthKHR: *hasResult = true; *hasResultType = true; break; + case OpConstantCompositeReplicateEXT: *hasResult = true; *hasResultType = true; break; + case OpSpecConstantCompositeReplicateEXT: *hasResult = true; *hasResultType = true; break; + case OpCompositeConstructReplicateEXT: *hasResult = true; *hasResultType = true; break; case OpTypeRayQueryKHR: *hasResult = true; *hasResultType = false; break; case OpRayQueryInitializeKHR: *hasResult = false; *hasResultType = false; break; case OpRayQueryTerminateKHR: *hasResult = false; *hasResultType = false; break; @@ -2267,6 +3055,18 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpRayQueryConfirmIntersectionKHR: *hasResult = false; *hasResultType = false; break; case OpRayQueryProceedKHR: *hasResult = true; *hasResultType = true; break; case OpRayQueryGetIntersectionTypeKHR: *hasResult = true; *hasResultType = true; break; + case OpImageSampleWeightedQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBoxFilterQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchSSDQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchSADQCOM: *hasResult = true; *hasResultType = true; break; + case OpBitCastArrayQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchWindowSSDQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchWindowSADQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchGatherSSDQCOM: *hasResult = true; *hasResultType = true; break; + case OpImageBlockMatchGatherSADQCOM: *hasResult = true; *hasResultType = true; break; + case OpCompositeConstructCoopMatQCOM: *hasResult = true; *hasResultType = true; break; + case OpCompositeExtractCoopMatQCOM: *hasResult = true; *hasResultType = true; break; + case OpExtractSubArrayQCOM: *hasResult = true; *hasResultType = true; break; case OpGroupIAddNonUniformAMD: *hasResult = true; *hasResultType = true; break; case OpGroupFAddNonUniformAMD: *hasResult = true; *hasResultType = true; break; case OpGroupFMinNonUniformAMD: *hasResult = true; *hasResultType = true; break; @@ -2278,19 +3078,121 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpFragmentMaskFetchAMD: *hasResult = true; *hasResultType = true; break; case OpFragmentFetchAMD: *hasResult = true; *hasResultType = true; break; case OpReadClockKHR: *hasResult = true; *hasResultType = true; break; + case OpAllocateNodePayloadsAMDX: *hasResult = true; *hasResultType = true; break; + case OpEnqueueNodePayloadsAMDX: *hasResult = false; *hasResultType = false; break; + case OpTypeNodePayloadArrayAMDX: *hasResult = true; *hasResultType = false; break; + case OpFinishWritingNodePayloadAMDX: *hasResult = true; *hasResultType = true; break; + case OpNodePayloadArrayLengthAMDX: *hasResult = true; *hasResultType = true; break; + case OpIsNodePayloadValidAMDX: *hasResult = true; *hasResultType = true; break; + case OpConstantStringAMDX: *hasResult = true; *hasResultType = false; break; + case OpSpecConstantStringAMDX: *hasResult = true; *hasResultType = false; break; + case OpGroupNonUniformQuadAllKHR: *hasResult = true; *hasResultType = true; break; + case OpGroupNonUniformQuadAnyKHR: *hasResult = true; *hasResultType = true; break; + case OpTypeBufferEXT: *hasResult = true; *hasResultType = false; break; + case OpBufferPointerEXT: *hasResult = true; *hasResultType = true; break; + case OpAbortKHR: *hasResult = false; *hasResultType = false; break; + case OpUntypedImageTexelPointerEXT: *hasResult = true; *hasResultType = true; break; + case OpMemberDecorateIdEXT: *hasResult = false; *hasResultType = false; break; + case OpConstantSizeOfEXT: *hasResult = true; *hasResultType = true; break; + case OpConstantDataKHR: *hasResult = true; *hasResultType = true; break; + case OpSpecConstantDataKHR: *hasResult = true; *hasResultType = true; break; + case OpPoisonKHR: *hasResult = true; *hasResultType = true; break; + case OpFreezeKHR: *hasResult = true; *hasResultType = true; break; + case OpHitObjectRecordHitMotionNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordHitWithIndexMotionNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordMissMotionNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetWorldToObjectNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectToWorldNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectRayDirectionNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectRayOriginNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectTraceRayMotionNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetShaderRecordBufferHandleNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetShaderBindingTableRecordIndexNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectRecordEmptyNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectTraceRayNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordHitNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordHitWithIndexNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordMissNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectExecuteShaderNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetCurrentTimeNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetAttributesNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetHitKindNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetPrimitiveIndexNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetGeometryIndexNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetInstanceIdNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetInstanceCustomIndexNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetWorldRayDirectionNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetWorldRayOriginNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetRayTMaxNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetRayTMinNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsEmptyNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsHitNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsMissNV: *hasResult = true; *hasResultType = true; break; + case OpReorderThreadWithHitObjectNV: *hasResult = false; *hasResultType = false; break; + case OpReorderThreadWithHintNV: *hasResult = false; *hasResultType = false; break; + case OpTypeHitObjectNV: *hasResult = true; *hasResultType = false; break; case OpImageSampleFootprintNV: *hasResult = true; *hasResultType = true; break; + case OpTypeVectorIdEXT: *hasResult = true; *hasResultType = false; break; + case OpCooperativeVectorMatrixMulNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeVectorOuterProductAccumulateNV: *hasResult = false; *hasResultType = false; break; + case OpCooperativeVectorReduceSumAccumulateNV: *hasResult = false; *hasResultType = false; break; + case OpCooperativeVectorMatrixMulAddNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixConvertNV: *hasResult = true; *hasResultType = true; break; case OpEmitMeshTasksEXT: *hasResult = false; *hasResultType = false; break; case OpSetMeshOutputsEXT: *hasResult = false; *hasResultType = false; break; - case OpGroupNonUniformPartitionNV: *hasResult = true; *hasResultType = true; break; + case OpGroupNonUniformPartitionEXT: *hasResult = true; *hasResultType = true; break; case OpWritePackedPrimitiveIndices4x8NV: *hasResult = false; *hasResultType = false; break; - case OpReportIntersectionNV: *hasResult = true; *hasResultType = true; break; + case OpFetchMicroTriangleVertexPositionNV: *hasResult = true; *hasResultType = true; break; + case OpFetchMicroTriangleVertexBarycentricNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeVectorLoadNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeVectorStoreNV: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordFromQueryEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordMissEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordMissMotionEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetIntersectionTriangleVertexPositionsEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetRayFlagsEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectSetShaderBindingTableRecordIndexEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectReorderExecuteShaderEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectTraceReorderExecuteEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectTraceMotionReorderExecuteEXT: *hasResult = false; *hasResultType = false; break; + case OpTypeHitObjectEXT: *hasResult = true; *hasResultType = false; break; + case OpReorderThreadWithHintEXT: *hasResult = false; *hasResultType = false; break; + case OpReorderThreadWithHitObjectEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectTraceRayEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectTraceRayMotionEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectRecordEmptyEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectExecuteShaderEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetCurrentTimeEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetAttributesEXT: *hasResult = false; *hasResultType = false; break; + case OpHitObjectGetHitKindEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetPrimitiveIndexEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetGeometryIndexEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetInstanceIdEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetInstanceCustomIndexEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectRayOriginEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectRayDirectionEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetWorldRayDirectionEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetWorldRayOriginEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetObjectToWorldEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetWorldToObjectEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetRayTMaxEXT: *hasResult = true; *hasResultType = true; break; + case OpReportIntersectionKHR: *hasResult = true; *hasResultType = true; break; case OpIgnoreIntersectionNV: *hasResult = false; *hasResultType = false; break; case OpTerminateRayNV: *hasResult = false; *hasResultType = false; break; case OpTraceNV: *hasResult = false; *hasResultType = false; break; case OpTraceMotionNV: *hasResult = false; *hasResultType = false; break; case OpTraceRayMotionNV: *hasResult = false; *hasResultType = false; break; - case OpTypeAccelerationStructureNV: *hasResult = true; *hasResultType = false; break; + case OpRayQueryGetIntersectionTriangleVertexPositionsKHR: *hasResult = true; *hasResultType = true; break; + case OpTypeAccelerationStructureKHR: *hasResult = true; *hasResultType = false; break; case OpExecuteCallableNV: *hasResult = false; *hasResultType = false; break; + case OpRayQueryGetIntersectionClusterIdNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetClusterIdNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetRayTMinEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetShaderBindingTableRecordIndexEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetShaderRecordBufferHandleEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsEmptyEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsHitEXT: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsMissEXT: *hasResult = true; *hasResultType = true; break; case OpTypeCooperativeMatrixNV: *hasResult = true; *hasResultType = false; break; case OpCooperativeMatrixLoadNV: *hasResult = true; *hasResultType = true; break; case OpCooperativeMatrixStoreNV: *hasResult = false; *hasResultType = false; break; @@ -2298,8 +3200,25 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpCooperativeMatrixLengthNV: *hasResult = true; *hasResultType = true; break; case OpBeginInvocationInterlockEXT: *hasResult = false; *hasResultType = false; break; case OpEndInvocationInterlockEXT: *hasResult = false; *hasResultType = false; break; + case OpCooperativeMatrixReduceNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixLoadTensorNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixStoreTensorNV: *hasResult = false; *hasResultType = false; break; + case OpCooperativeMatrixPerElementOpNV: *hasResult = true; *hasResultType = true; break; + case OpTypeTensorLayoutNV: *hasResult = true; *hasResultType = false; break; + case OpTypeTensorViewNV: *hasResult = true; *hasResultType = false; break; + case OpCreateTensorLayoutNV: *hasResult = true; *hasResultType = true; break; + case OpTensorLayoutSetDimensionNV: *hasResult = true; *hasResultType = true; break; + case OpTensorLayoutSetStrideNV: *hasResult = true; *hasResultType = true; break; + case OpTensorLayoutSliceNV: *hasResult = true; *hasResultType = true; break; + case OpTensorLayoutSetClampValueNV: *hasResult = true; *hasResultType = true; break; + case OpCreateTensorViewNV: *hasResult = true; *hasResultType = true; break; + case OpTensorViewSetDimensionNV: *hasResult = true; *hasResultType = true; break; + case OpTensorViewSetStrideNV: *hasResult = true; *hasResultType = true; break; case OpDemoteToHelperInvocation: *hasResult = false; *hasResultType = false; break; case OpIsHelperInvocationEXT: *hasResult = true; *hasResultType = true; break; + case OpTensorViewSetClipNV: *hasResult = true; *hasResultType = true; break; + case OpTensorLayoutSetBlockSizeNV: *hasResult = true; *hasResultType = true; break; + case OpCooperativeMatrixTransposeNV: *hasResult = true; *hasResultType = true; break; case OpConvertUToImageNV: *hasResult = true; *hasResultType = true; break; case OpConvertUToSamplerNV: *hasResult = true; *hasResultType = true; break; case OpConvertImageToUNV: *hasResult = true; *hasResultType = true; break; @@ -2307,6 +3226,20 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpConvertUToSampledImageNV: *hasResult = true; *hasResultType = true; break; case OpConvertSampledImageToUNV: *hasResult = true; *hasResultType = true; break; case OpSamplerImageAddressingModeNV: *hasResult = false; *hasResultType = false; break; + case OpRawAccessChainNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryGetIntersectionSpherePositionNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryGetIntersectionSphereRadiusNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryGetIntersectionLSSPositionsNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryGetIntersectionLSSRadiiNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryGetIntersectionLSSHitValueNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetSpherePositionNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetSphereRadiusNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetLSSPositionsNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectGetLSSRadiiNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsSphereHitNV: *hasResult = true; *hasResultType = true; break; + case OpHitObjectIsLSSHitNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryIsSphereHitNV: *hasResult = true; *hasResultType = true; break; + case OpRayQueryIsLSSHitNV: *hasResult = true; *hasResultType = true; break; case OpSubgroupShuffleINTEL: *hasResult = true; *hasResultType = true; break; case OpSubgroupShuffleDownINTEL: *hasResult = true; *hasResultType = true; break; case OpSubgroupShuffleUpINTEL: *hasResult = true; *hasResultType = true; break; @@ -2333,7 +3266,7 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpUMul32x16INTEL: *hasResult = true; *hasResultType = true; break; case OpConstantFunctionPointerINTEL: *hasResult = true; *hasResultType = true; break; case OpFunctionPointerCallINTEL: *hasResult = true; *hasResultType = true; break; - case OpAsmTargetINTEL: *hasResult = true; *hasResultType = true; break; + case OpAsmTargetINTEL: *hasResult = true; *hasResultType = false; break; case OpAsmINTEL: *hasResult = true; *hasResultType = true; break; case OpAsmCallINTEL: *hasResult = true; *hasResultType = true; break; case OpAtomicFMinEXT: *hasResult = true; *hasResultType = true; break; @@ -2463,24 +3396,24 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpVariableLengthArrayINTEL: *hasResult = true; *hasResultType = true; break; case OpSaveMemoryINTEL: *hasResult = true; *hasResultType = true; break; case OpRestoreMemoryINTEL: *hasResult = false; *hasResultType = false; break; - case OpArbitraryFloatSinCosPiINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatCastINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatCastFromIntINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatCastToIntINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatAddINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatSubINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatMulINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatDivINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatGTINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatGEINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatLTINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatLEINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatEQINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatRecipINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatRSqrtINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatCbrtINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatHypotINTEL: *hasResult = true; *hasResultType = true; break; - case OpArbitraryFloatSqrtINTEL: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatSinCosPiALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatCastALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatCastFromIntALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatCastToIntALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatAddALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatSubALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatMulALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatDivALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatGTALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatGEALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatLTALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatLEALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatEQALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatRecipALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatRSqrtALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatCbrtALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatHypotALTERA: *hasResult = true; *hasResultType = true; break; + case OpArbitraryFloatSqrtALTERA: *hasResult = true; *hasResultType = true; break; case OpArbitraryFloatLogINTEL: *hasResult = true; *hasResultType = true; break; case OpArbitraryFloatLog2INTEL: *hasResult = true; *hasResultType = true; break; case OpArbitraryFloatLog10INTEL: *hasResult = true; *hasResultType = true; break; @@ -2508,22 +3441,22 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpAliasDomainDeclINTEL: *hasResult = true; *hasResultType = false; break; case OpAliasScopeDeclINTEL: *hasResult = true; *hasResultType = false; break; case OpAliasScopeListDeclINTEL: *hasResult = true; *hasResultType = false; break; - case OpFixedSqrtINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedRecipINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedRsqrtINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedSinINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedCosINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedSinCosINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedSinPiINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedCosPiINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedSinCosPiINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedLogINTEL: *hasResult = true; *hasResultType = true; break; - case OpFixedExpINTEL: *hasResult = true; *hasResultType = true; break; - case OpPtrCastToCrossWorkgroupINTEL: *hasResult = true; *hasResultType = true; break; - case OpCrossWorkgroupCastToPtrINTEL: *hasResult = true; *hasResultType = true; break; - case OpReadPipeBlockingINTEL: *hasResult = true; *hasResultType = true; break; - case OpWritePipeBlockingINTEL: *hasResult = true; *hasResultType = true; break; - case OpFPGARegINTEL: *hasResult = true; *hasResultType = true; break; + case OpFixedSqrtALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedRecipALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedRsqrtALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedSinALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedCosALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedSinCosALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedSinPiALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedCosPiALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedSinCosPiALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedLogALTERA: *hasResult = true; *hasResultType = true; break; + case OpFixedExpALTERA: *hasResult = true; *hasResultType = true; break; + case OpPtrCastToCrossWorkgroupALTERA: *hasResult = true; *hasResultType = true; break; + case OpCrossWorkgroupCastToPtrALTERA: *hasResult = true; *hasResultType = true; break; + case OpReadPipeBlockingALTERA: *hasResult = true; *hasResultType = true; break; + case OpWritePipeBlockingALTERA: *hasResult = true; *hasResultType = true; break; + case OpFPGARegALTERA: *hasResult = true; *hasResultType = true; break; case OpRayQueryGetRayTMinKHR: *hasResult = true; *hasResultType = true; break; case OpRayQueryGetRayFlagsKHR: *hasResult = true; *hasResultType = true; break; case OpRayQueryGetIntersectionTKHR: *hasResult = true; *hasResultType = true; break; @@ -2546,8 +3479,33 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpTypeStructContinuedINTEL: *hasResult = false; *hasResultType = false; break; case OpConstantCompositeContinuedINTEL: *hasResult = false; *hasResultType = false; break; case OpSpecConstantCompositeContinuedINTEL: *hasResult = false; *hasResultType = false; break; + case OpCompositeConstructContinuedINTEL: *hasResult = true; *hasResultType = true; break; + case OpConvertFToBF16INTEL: *hasResult = true; *hasResultType = true; break; + case OpConvertBF16ToFINTEL: *hasResult = true; *hasResultType = true; break; case OpControlBarrierArriveINTEL: *hasResult = false; *hasResultType = false; break; case OpControlBarrierWaitINTEL: *hasResult = false; *hasResultType = false; break; + case OpArithmeticFenceEXT: *hasResult = true; *hasResultType = true; break; + case OpTaskSequenceCreateALTERA: *hasResult = true; *hasResultType = true; break; + case OpTaskSequenceAsyncALTERA: *hasResult = false; *hasResultType = false; break; + case OpTaskSequenceGetALTERA: *hasResult = true; *hasResultType = true; break; + case OpTaskSequenceReleaseALTERA: *hasResult = false; *hasResultType = false; break; + case OpTypeTaskSequenceALTERA: *hasResult = true; *hasResultType = false; break; + case OpSubgroupBlockPrefetchINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroup2DBlockLoadINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroup2DBlockLoadTransformINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroup2DBlockLoadTransposeINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroup2DBlockPrefetchINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroup2DBlockStoreINTEL: *hasResult = false; *hasResultType = false; break; + case OpSubgroupMatrixMultiplyAccumulateINTEL: *hasResult = true; *hasResultType = true; break; + case OpBitwiseFunctionINTEL: *hasResult = true; *hasResultType = true; break; + case OpUntypedVariableLengthArrayINTEL: *hasResult = true; *hasResultType = true; break; + case OpConditionalExtensionINTEL: *hasResult = false; *hasResultType = false; break; + case OpConditionalEntryPointINTEL: *hasResult = false; *hasResultType = false; break; + case OpConditionalCapabilityINTEL: *hasResult = false; *hasResultType = false; break; + case OpSpecConstantTargetINTEL: *hasResult = true; *hasResultType = true; break; + case OpSpecConstantArchitectureINTEL: *hasResult = true; *hasResultType = true; break; + case OpSpecConstantCapabilitiesINTEL: *hasResult = true; *hasResultType = true; break; + case OpConditionalCopyObjectINTEL: *hasResult = true; *hasResultType = true; break; case OpGroupIMulKHR: *hasResult = true; *hasResultType = true; break; case OpGroupFMulKHR: *hasResult = true; *hasResultType = true; break; case OpGroupBitwiseAndKHR: *hasResult = true; *hasResultType = true; break; @@ -2556,22 +3514,2152 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) { case OpGroupLogicalAndKHR: *hasResult = true; *hasResultType = true; break; case OpGroupLogicalOrKHR: *hasResult = true; *hasResultType = true; break; case OpGroupLogicalXorKHR: *hasResult = true; *hasResultType = true; break; + case OpRoundFToTF32INTEL: *hasResult = true; *hasResultType = true; break; + case OpMaskedGatherINTEL: *hasResult = true; *hasResultType = true; break; + case OpMaskedScatterINTEL: *hasResult = false; *hasResultType = false; break; + case OpConvertHandleToImageINTEL: *hasResult = true; *hasResultType = true; break; + case OpConvertHandleToSamplerINTEL: *hasResult = true; *hasResultType = true; break; + case OpConvertHandleToSampledImageINTEL: *hasResult = true; *hasResultType = true; break; + case OpFDot2MixAcc32VALVE: *hasResult = true; *hasResultType = true; break; + case OpFDot2MixAcc16VALVE: *hasResult = true; *hasResultType = true; break; + case OpFDot4MixAcc32VALVE: *hasResult = true; *hasResultType = true; break; } } +inline const char* SourceLanguageToString(SourceLanguage value) { + switch (value) { + case SourceLanguageUnknown: return "Unknown"; + case SourceLanguageESSL: return "ESSL"; + case SourceLanguageGLSL: return "GLSL"; + case SourceLanguageOpenCL_C: return "OpenCL_C"; + case SourceLanguageOpenCL_CPP: return "OpenCL_CPP"; + case SourceLanguageHLSL: return "HLSL"; + case SourceLanguageCPP_for_OpenCL: return "CPP_for_OpenCL"; + case SourceLanguageSYCL: return "SYCL"; + case SourceLanguageHERO_C: return "HERO_C"; + case SourceLanguageNZSL: return "NZSL"; + case SourceLanguageWGSL: return "WGSL"; + case SourceLanguageSlang: return "Slang"; + case SourceLanguageZig: return "Zig"; + case SourceLanguageRust: return "Rust"; + default: return "Unknown"; + } +} + +inline const char* ExecutionModelToString(ExecutionModel value) { + switch (value) { + case ExecutionModelVertex: return "Vertex"; + case ExecutionModelTessellationControl: return "TessellationControl"; + case ExecutionModelTessellationEvaluation: return "TessellationEvaluation"; + case ExecutionModelGeometry: return "Geometry"; + case ExecutionModelFragment: return "Fragment"; + case ExecutionModelGLCompute: return "GLCompute"; + case ExecutionModelKernel: return "Kernel"; + case ExecutionModelTaskNV: return "TaskNV"; + case ExecutionModelMeshNV: return "MeshNV"; + case ExecutionModelRayGenerationKHR: return "RayGenerationKHR"; + case ExecutionModelIntersectionKHR: return "IntersectionKHR"; + case ExecutionModelAnyHitKHR: return "AnyHitKHR"; + case ExecutionModelClosestHitKHR: return "ClosestHitKHR"; + case ExecutionModelMissKHR: return "MissKHR"; + case ExecutionModelCallableKHR: return "CallableKHR"; + case ExecutionModelTaskEXT: return "TaskEXT"; + case ExecutionModelMeshEXT: return "MeshEXT"; + default: return "Unknown"; + } +} + +inline const char* AddressingModelToString(AddressingModel value) { + switch (value) { + case AddressingModelLogical: return "Logical"; + case AddressingModelPhysical32: return "Physical32"; + case AddressingModelPhysical64: return "Physical64"; + case AddressingModelPhysicalStorageBuffer64: return "PhysicalStorageBuffer64"; + default: return "Unknown"; + } +} + +inline const char* MemoryModelToString(MemoryModel value) { + switch (value) { + case MemoryModelSimple: return "Simple"; + case MemoryModelGLSL450: return "GLSL450"; + case MemoryModelOpenCL: return "OpenCL"; + case MemoryModelVulkan: return "Vulkan"; + default: return "Unknown"; + } +} + +inline const char* ExecutionModeToString(ExecutionMode value) { + switch (value) { + case ExecutionModeInvocations: return "Invocations"; + case ExecutionModeSpacingEqual: return "SpacingEqual"; + case ExecutionModeSpacingFractionalEven: return "SpacingFractionalEven"; + case ExecutionModeSpacingFractionalOdd: return "SpacingFractionalOdd"; + case ExecutionModeVertexOrderCw: return "VertexOrderCw"; + case ExecutionModeVertexOrderCcw: return "VertexOrderCcw"; + case ExecutionModePixelCenterInteger: return "PixelCenterInteger"; + case ExecutionModeOriginUpperLeft: return "OriginUpperLeft"; + case ExecutionModeOriginLowerLeft: return "OriginLowerLeft"; + case ExecutionModeEarlyFragmentTests: return "EarlyFragmentTests"; + case ExecutionModePointMode: return "PointMode"; + case ExecutionModeXfb: return "Xfb"; + case ExecutionModeDepthReplacing: return "DepthReplacing"; + case ExecutionModeDepthGreater: return "DepthGreater"; + case ExecutionModeDepthLess: return "DepthLess"; + case ExecutionModeDepthUnchanged: return "DepthUnchanged"; + case ExecutionModeLocalSize: return "LocalSize"; + case ExecutionModeLocalSizeHint: return "LocalSizeHint"; + case ExecutionModeInputPoints: return "InputPoints"; + case ExecutionModeInputLines: return "InputLines"; + case ExecutionModeInputLinesAdjacency: return "InputLinesAdjacency"; + case ExecutionModeTriangles: return "Triangles"; + case ExecutionModeInputTrianglesAdjacency: return "InputTrianglesAdjacency"; + case ExecutionModeQuads: return "Quads"; + case ExecutionModeIsolines: return "Isolines"; + case ExecutionModeOutputVertices: return "OutputVertices"; + case ExecutionModeOutputPoints: return "OutputPoints"; + case ExecutionModeOutputLineStrip: return "OutputLineStrip"; + case ExecutionModeOutputTriangleStrip: return "OutputTriangleStrip"; + case ExecutionModeVecTypeHint: return "VecTypeHint"; + case ExecutionModeContractionOff: return "ContractionOff"; + case ExecutionModeInitializer: return "Initializer"; + case ExecutionModeFinalizer: return "Finalizer"; + case ExecutionModeSubgroupSize: return "SubgroupSize"; + case ExecutionModeSubgroupsPerWorkgroup: return "SubgroupsPerWorkgroup"; + case ExecutionModeSubgroupsPerWorkgroupId: return "SubgroupsPerWorkgroupId"; + case ExecutionModeLocalSizeId: return "LocalSizeId"; + case ExecutionModeLocalSizeHintId: return "LocalSizeHintId"; + case ExecutionModeNonCoherentColorAttachmentReadEXT: return "NonCoherentColorAttachmentReadEXT"; + case ExecutionModeNonCoherentDepthAttachmentReadEXT: return "NonCoherentDepthAttachmentReadEXT"; + case ExecutionModeNonCoherentStencilAttachmentReadEXT: return "NonCoherentStencilAttachmentReadEXT"; + case ExecutionModeSubgroupUniformControlFlowKHR: return "SubgroupUniformControlFlowKHR"; + case ExecutionModePostDepthCoverage: return "PostDepthCoverage"; + case ExecutionModeDenormPreserve: return "DenormPreserve"; + case ExecutionModeDenormFlushToZero: return "DenormFlushToZero"; + case ExecutionModeSignedZeroInfNanPreserve: return "SignedZeroInfNanPreserve"; + case ExecutionModeRoundingModeRTE: return "RoundingModeRTE"; + case ExecutionModeRoundingModeRTZ: return "RoundingModeRTZ"; + case ExecutionModeNonCoherentTileAttachmentReadQCOM: return "NonCoherentTileAttachmentReadQCOM"; + case ExecutionModeTileShadingRateQCOM: return "TileShadingRateQCOM"; + case ExecutionModeEarlyAndLateFragmentTestsAMD: return "EarlyAndLateFragmentTestsAMD"; + case ExecutionModeStencilRefReplacingEXT: return "StencilRefReplacingEXT"; + case ExecutionModeCoalescingAMDX: return "CoalescingAMDX"; + case ExecutionModeIsApiEntryAMDX: return "IsApiEntryAMDX"; + case ExecutionModeMaxNodeRecursionAMDX: return "MaxNodeRecursionAMDX"; + case ExecutionModeStaticNumWorkgroupsAMDX: return "StaticNumWorkgroupsAMDX"; + case ExecutionModeShaderIndexAMDX: return "ShaderIndexAMDX"; + case ExecutionModeMaxNumWorkgroupsAMDX: return "MaxNumWorkgroupsAMDX"; + case ExecutionModeStencilRefUnchangedFrontAMD: return "StencilRefUnchangedFrontAMD"; + case ExecutionModeStencilRefGreaterFrontAMD: return "StencilRefGreaterFrontAMD"; + case ExecutionModeStencilRefLessFrontAMD: return "StencilRefLessFrontAMD"; + case ExecutionModeStencilRefUnchangedBackAMD: return "StencilRefUnchangedBackAMD"; + case ExecutionModeStencilRefGreaterBackAMD: return "StencilRefGreaterBackAMD"; + case ExecutionModeStencilRefLessBackAMD: return "StencilRefLessBackAMD"; + case ExecutionModeQuadDerivativesKHR: return "QuadDerivativesKHR"; + case ExecutionModeRequireFullQuadsKHR: return "RequireFullQuadsKHR"; + case ExecutionModeSharesInputWithAMDX: return "SharesInputWithAMDX"; + case ExecutionModeArithmeticPoisonKHR: return "ArithmeticPoisonKHR"; + case ExecutionModeOutputLinesEXT: return "OutputLinesEXT"; + case ExecutionModeOutputPrimitivesEXT: return "OutputPrimitivesEXT"; + case ExecutionModeDerivativeGroupQuadsKHR: return "DerivativeGroupQuadsKHR"; + case ExecutionModeDerivativeGroupLinearKHR: return "DerivativeGroupLinearKHR"; + case ExecutionModeOutputTrianglesEXT: return "OutputTrianglesEXT"; + case ExecutionModePixelInterlockOrderedEXT: return "PixelInterlockOrderedEXT"; + case ExecutionModePixelInterlockUnorderedEXT: return "PixelInterlockUnorderedEXT"; + case ExecutionModeSampleInterlockOrderedEXT: return "SampleInterlockOrderedEXT"; + case ExecutionModeSampleInterlockUnorderedEXT: return "SampleInterlockUnorderedEXT"; + case ExecutionModeShadingRateInterlockOrderedEXT: return "ShadingRateInterlockOrderedEXT"; + case ExecutionModeShadingRateInterlockUnorderedEXT: return "ShadingRateInterlockUnorderedEXT"; + case ExecutionModeShader64BitIndexingEXT: return "Shader64BitIndexingEXT"; + case ExecutionModeSharedLocalMemorySizeINTEL: return "SharedLocalMemorySizeINTEL"; + case ExecutionModeRoundingModeRTPINTEL: return "RoundingModeRTPINTEL"; + case ExecutionModeRoundingModeRTNINTEL: return "RoundingModeRTNINTEL"; + case ExecutionModeFloatingPointModeALTINTEL: return "FloatingPointModeALTINTEL"; + case ExecutionModeFloatingPointModeIEEEINTEL: return "FloatingPointModeIEEEINTEL"; + case ExecutionModeMaxWorkgroupSizeINTEL: return "MaxWorkgroupSizeINTEL"; + case ExecutionModeMaxWorkDimINTEL: return "MaxWorkDimINTEL"; + case ExecutionModeNoGlobalOffsetINTEL: return "NoGlobalOffsetINTEL"; + case ExecutionModeNumSIMDWorkitemsINTEL: return "NumSIMDWorkitemsINTEL"; + case ExecutionModeSchedulerTargetFmaxMhzINTEL: return "SchedulerTargetFmaxMhzINTEL"; + case ExecutionModeMaximallyReconvergesKHR: return "MaximallyReconvergesKHR"; + case ExecutionModeFPFastMathDefault: return "FPFastMathDefault"; + case ExecutionModeStreamingInterfaceINTEL: return "StreamingInterfaceINTEL"; + case ExecutionModeRegisterMapInterfaceINTEL: return "RegisterMapInterfaceINTEL"; + case ExecutionModeNamedBarrierCountINTEL: return "NamedBarrierCountINTEL"; + case ExecutionModeMaximumRegistersINTEL: return "MaximumRegistersINTEL"; + case ExecutionModeMaximumRegistersIdINTEL: return "MaximumRegistersIdINTEL"; + case ExecutionModeNamedMaximumRegistersINTEL: return "NamedMaximumRegistersINTEL"; + default: return "Unknown"; + } +} + +inline const char* StorageClassToString(StorageClass value) { + switch (value) { + case StorageClassUniformConstant: return "UniformConstant"; + case StorageClassInput: return "Input"; + case StorageClassUniform: return "Uniform"; + case StorageClassOutput: return "Output"; + case StorageClassWorkgroup: return "Workgroup"; + case StorageClassCrossWorkgroup: return "CrossWorkgroup"; + case StorageClassPrivate: return "Private"; + case StorageClassFunction: return "Function"; + case StorageClassGeneric: return "Generic"; + case StorageClassPushConstant: return "PushConstant"; + case StorageClassAtomicCounter: return "AtomicCounter"; + case StorageClassImage: return "Image"; + case StorageClassStorageBuffer: return "StorageBuffer"; + case StorageClassTileImageEXT: return "TileImageEXT"; + case StorageClassTileAttachmentQCOM: return "TileAttachmentQCOM"; + case StorageClassNodePayloadAMDX: return "NodePayloadAMDX"; + case StorageClassCallableDataKHR: return "CallableDataKHR"; + case StorageClassIncomingCallableDataKHR: return "IncomingCallableDataKHR"; + case StorageClassRayPayloadKHR: return "RayPayloadKHR"; + case StorageClassHitAttributeKHR: return "HitAttributeKHR"; + case StorageClassIncomingRayPayloadKHR: return "IncomingRayPayloadKHR"; + case StorageClassShaderRecordBufferKHR: return "ShaderRecordBufferKHR"; + case StorageClassPhysicalStorageBuffer: return "PhysicalStorageBuffer"; + case StorageClassHitObjectAttributeNV: return "HitObjectAttributeNV"; + case StorageClassTaskPayloadWorkgroupEXT: return "TaskPayloadWorkgroupEXT"; + case StorageClassHitObjectAttributeEXT: return "HitObjectAttributeEXT"; + case StorageClassCodeSectionINTEL: return "CodeSectionINTEL"; + case StorageClassDeviceOnlyALTERA: return "DeviceOnlyALTERA"; + case StorageClassHostOnlyALTERA: return "HostOnlyALTERA"; + default: return "Unknown"; + } +} + +inline const char* DimToString(Dim value) { + switch (value) { + case Dim1D: return "1D"; + case Dim2D: return "2D"; + case Dim3D: return "3D"; + case DimCube: return "Cube"; + case DimRect: return "Rect"; + case DimBuffer: return "Buffer"; + case DimSubpassData: return "SubpassData"; + case DimTileImageDataEXT: return "TileImageDataEXT"; + default: return "Unknown"; + } +} + +inline const char* SamplerAddressingModeToString(SamplerAddressingMode value) { + switch (value) { + case SamplerAddressingModeNone: return "None"; + case SamplerAddressingModeClampToEdge: return "ClampToEdge"; + case SamplerAddressingModeClamp: return "Clamp"; + case SamplerAddressingModeRepeat: return "Repeat"; + case SamplerAddressingModeRepeatMirrored: return "RepeatMirrored"; + default: return "Unknown"; + } +} + +inline const char* SamplerFilterModeToString(SamplerFilterMode value) { + switch (value) { + case SamplerFilterModeNearest: return "Nearest"; + case SamplerFilterModeLinear: return "Linear"; + default: return "Unknown"; + } +} + +inline const char* ImageFormatToString(ImageFormat value) { + switch (value) { + case ImageFormatUnknown: return "Unknown"; + case ImageFormatRgba32f: return "Rgba32f"; + case ImageFormatRgba16f: return "Rgba16f"; + case ImageFormatR32f: return "R32f"; + case ImageFormatRgba8: return "Rgba8"; + case ImageFormatRgba8Snorm: return "Rgba8Snorm"; + case ImageFormatRg32f: return "Rg32f"; + case ImageFormatRg16f: return "Rg16f"; + case ImageFormatR11fG11fB10f: return "R11fG11fB10f"; + case ImageFormatR16f: return "R16f"; + case ImageFormatRgba16: return "Rgba16"; + case ImageFormatRgb10A2: return "Rgb10A2"; + case ImageFormatRg16: return "Rg16"; + case ImageFormatRg8: return "Rg8"; + case ImageFormatR16: return "R16"; + case ImageFormatR8: return "R8"; + case ImageFormatRgba16Snorm: return "Rgba16Snorm"; + case ImageFormatRg16Snorm: return "Rg16Snorm"; + case ImageFormatRg8Snorm: return "Rg8Snorm"; + case ImageFormatR16Snorm: return "R16Snorm"; + case ImageFormatR8Snorm: return "R8Snorm"; + case ImageFormatRgba32i: return "Rgba32i"; + case ImageFormatRgba16i: return "Rgba16i"; + case ImageFormatRgba8i: return "Rgba8i"; + case ImageFormatR32i: return "R32i"; + case ImageFormatRg32i: return "Rg32i"; + case ImageFormatRg16i: return "Rg16i"; + case ImageFormatRg8i: return "Rg8i"; + case ImageFormatR16i: return "R16i"; + case ImageFormatR8i: return "R8i"; + case ImageFormatRgba32ui: return "Rgba32ui"; + case ImageFormatRgba16ui: return "Rgba16ui"; + case ImageFormatRgba8ui: return "Rgba8ui"; + case ImageFormatR32ui: return "R32ui"; + case ImageFormatRgb10a2ui: return "Rgb10a2ui"; + case ImageFormatRg32ui: return "Rg32ui"; + case ImageFormatRg16ui: return "Rg16ui"; + case ImageFormatRg8ui: return "Rg8ui"; + case ImageFormatR16ui: return "R16ui"; + case ImageFormatR8ui: return "R8ui"; + case ImageFormatR64ui: return "R64ui"; + case ImageFormatR64i: return "R64i"; + default: return "Unknown"; + } +} + +inline const char* ImageChannelOrderToString(ImageChannelOrder value) { + switch (value) { + case ImageChannelOrderR: return "R"; + case ImageChannelOrderA: return "A"; + case ImageChannelOrderRG: return "RG"; + case ImageChannelOrderRA: return "RA"; + case ImageChannelOrderRGB: return "RGB"; + case ImageChannelOrderRGBA: return "RGBA"; + case ImageChannelOrderBGRA: return "BGRA"; + case ImageChannelOrderARGB: return "ARGB"; + case ImageChannelOrderIntensity: return "Intensity"; + case ImageChannelOrderLuminance: return "Luminance"; + case ImageChannelOrderRx: return "Rx"; + case ImageChannelOrderRGx: return "RGx"; + case ImageChannelOrderRGBx: return "RGBx"; + case ImageChannelOrderDepth: return "Depth"; + case ImageChannelOrderDepthStencil: return "DepthStencil"; + case ImageChannelOrdersRGB: return "sRGB"; + case ImageChannelOrdersRGBx: return "sRGBx"; + case ImageChannelOrdersRGBA: return "sRGBA"; + case ImageChannelOrdersBGRA: return "sBGRA"; + case ImageChannelOrderABGR: return "ABGR"; + default: return "Unknown"; + } +} + +inline const char* ImageChannelDataTypeToString(ImageChannelDataType value) { + switch (value) { + case ImageChannelDataTypeSnormInt8: return "SnormInt8"; + case ImageChannelDataTypeSnormInt16: return "SnormInt16"; + case ImageChannelDataTypeUnormInt8: return "UnormInt8"; + case ImageChannelDataTypeUnormInt16: return "UnormInt16"; + case ImageChannelDataTypeUnormShort565: return "UnormShort565"; + case ImageChannelDataTypeUnormShort555: return "UnormShort555"; + case ImageChannelDataTypeUnormInt101010: return "UnormInt101010"; + case ImageChannelDataTypeSignedInt8: return "SignedInt8"; + case ImageChannelDataTypeSignedInt16: return "SignedInt16"; + case ImageChannelDataTypeSignedInt32: return "SignedInt32"; + case ImageChannelDataTypeUnsignedInt8: return "UnsignedInt8"; + case ImageChannelDataTypeUnsignedInt16: return "UnsignedInt16"; + case ImageChannelDataTypeUnsignedInt32: return "UnsignedInt32"; + case ImageChannelDataTypeHalfFloat: return "HalfFloat"; + case ImageChannelDataTypeFloat: return "Float"; + case ImageChannelDataTypeUnormInt24: return "UnormInt24"; + case ImageChannelDataTypeUnormInt101010_2: return "UnormInt101010_2"; + case ImageChannelDataTypeUnormInt10X6EXT: return "UnormInt10X6EXT"; + case ImageChannelDataTypeUnsignedIntRaw10EXT: return "UnsignedIntRaw10EXT"; + case ImageChannelDataTypeUnsignedIntRaw12EXT: return "UnsignedIntRaw12EXT"; + case ImageChannelDataTypeUnormInt2_101010EXT: return "UnormInt2_101010EXT"; + case ImageChannelDataTypeUnsignedInt10X6EXT: return "UnsignedInt10X6EXT"; + case ImageChannelDataTypeUnsignedInt12X4EXT: return "UnsignedInt12X4EXT"; + case ImageChannelDataTypeUnsignedInt14X2EXT: return "UnsignedInt14X2EXT"; + case ImageChannelDataTypeUnormInt12X4EXT: return "UnormInt12X4EXT"; + case ImageChannelDataTypeUnormInt14X2EXT: return "UnormInt14X2EXT"; + default: return "Unknown"; + } +} + +inline const char* FPRoundingModeToString(FPRoundingMode value) { + switch (value) { + case FPRoundingModeRTE: return "RTE"; + case FPRoundingModeRTZ: return "RTZ"; + case FPRoundingModeRTP: return "RTP"; + case FPRoundingModeRTN: return "RTN"; + default: return "Unknown"; + } +} + +inline const char* LinkageTypeToString(LinkageType value) { + switch (value) { + case LinkageTypeExport: return "Export"; + case LinkageTypeImport: return "Import"; + case LinkageTypeLinkOnceODR: return "LinkOnceODR"; + default: return "Unknown"; + } +} + +inline const char* AccessQualifierToString(AccessQualifier value) { + switch (value) { + case AccessQualifierReadOnly: return "ReadOnly"; + case AccessQualifierWriteOnly: return "WriteOnly"; + case AccessQualifierReadWrite: return "ReadWrite"; + default: return "Unknown"; + } +} + +inline const char* FunctionParameterAttributeToString(FunctionParameterAttribute value) { + switch (value) { + case FunctionParameterAttributeZext: return "Zext"; + case FunctionParameterAttributeSext: return "Sext"; + case FunctionParameterAttributeByVal: return "ByVal"; + case FunctionParameterAttributeSret: return "Sret"; + case FunctionParameterAttributeNoAlias: return "NoAlias"; + case FunctionParameterAttributeNoCapture: return "NoCapture"; + case FunctionParameterAttributeNoWrite: return "NoWrite"; + case FunctionParameterAttributeNoReadWrite: return "NoReadWrite"; + case FunctionParameterAttributeRuntimeAlignedALTERA: return "RuntimeAlignedALTERA"; + default: return "Unknown"; + } +} + +inline const char* DecorationToString(Decoration value) { + switch (value) { + case DecorationRelaxedPrecision: return "RelaxedPrecision"; + case DecorationSpecId: return "SpecId"; + case DecorationBlock: return "Block"; + case DecorationBufferBlock: return "BufferBlock"; + case DecorationRowMajor: return "RowMajor"; + case DecorationColMajor: return "ColMajor"; + case DecorationArrayStride: return "ArrayStride"; + case DecorationMatrixStride: return "MatrixStride"; + case DecorationGLSLShared: return "GLSLShared"; + case DecorationGLSLPacked: return "GLSLPacked"; + case DecorationCPacked: return "CPacked"; + case DecorationBuiltIn: return "BuiltIn"; + case DecorationNoPerspective: return "NoPerspective"; + case DecorationFlat: return "Flat"; + case DecorationPatch: return "Patch"; + case DecorationCentroid: return "Centroid"; + case DecorationSample: return "Sample"; + case DecorationInvariant: return "Invariant"; + case DecorationRestrict: return "Restrict"; + case DecorationAliased: return "Aliased"; + case DecorationVolatile: return "Volatile"; + case DecorationConstant: return "Constant"; + case DecorationCoherent: return "Coherent"; + case DecorationNonWritable: return "NonWritable"; + case DecorationNonReadable: return "NonReadable"; + case DecorationUniform: return "Uniform"; + case DecorationUniformId: return "UniformId"; + case DecorationSaturatedConversion: return "SaturatedConversion"; + case DecorationStream: return "Stream"; + case DecorationLocation: return "Location"; + case DecorationComponent: return "Component"; + case DecorationIndex: return "Index"; + case DecorationBinding: return "Binding"; + case DecorationDescriptorSet: return "DescriptorSet"; + case DecorationOffset: return "Offset"; + case DecorationXfbBuffer: return "XfbBuffer"; + case DecorationXfbStride: return "XfbStride"; + case DecorationFuncParamAttr: return "FuncParamAttr"; + case DecorationFPRoundingMode: return "FPRoundingMode"; + case DecorationFPFastMathMode: return "FPFastMathMode"; + case DecorationLinkageAttributes: return "LinkageAttributes"; + case DecorationNoContraction: return "NoContraction"; + case DecorationInputAttachmentIndex: return "InputAttachmentIndex"; + case DecorationAlignment: return "Alignment"; + case DecorationMaxByteOffset: return "MaxByteOffset"; + case DecorationAlignmentId: return "AlignmentId"; + case DecorationMaxByteOffsetId: return "MaxByteOffsetId"; + case DecorationSaturatedToLargestFloat8NormalConversionEXT: return "SaturatedToLargestFloat8NormalConversionEXT"; + case DecorationNoSignedWrap: return "NoSignedWrap"; + case DecorationNoUnsignedWrap: return "NoUnsignedWrap"; + case DecorationWeightTextureQCOM: return "WeightTextureQCOM"; + case DecorationBlockMatchTextureQCOM: return "BlockMatchTextureQCOM"; + case DecorationBlockMatchSamplerQCOM: return "BlockMatchSamplerQCOM"; + case DecorationExplicitInterpAMD: return "ExplicitInterpAMD"; + case DecorationNodeSharesPayloadLimitsWithAMDX: return "NodeSharesPayloadLimitsWithAMDX"; + case DecorationNodeMaxPayloadsAMDX: return "NodeMaxPayloadsAMDX"; + case DecorationTrackFinishWritingAMDX: return "TrackFinishWritingAMDX"; + case DecorationPayloadNodeNameAMDX: return "PayloadNodeNameAMDX"; + case DecorationPayloadNodeBaseIndexAMDX: return "PayloadNodeBaseIndexAMDX"; + case DecorationPayloadNodeSparseArrayAMDX: return "PayloadNodeSparseArrayAMDX"; + case DecorationPayloadNodeArraySizeAMDX: return "PayloadNodeArraySizeAMDX"; + case DecorationPayloadDispatchIndirectAMDX: return "PayloadDispatchIndirectAMDX"; + case DecorationArrayStrideIdEXT: return "ArrayStrideIdEXT"; + case DecorationOffsetIdEXT: return "OffsetIdEXT"; + case DecorationUTFEncodedKHR: return "UTFEncodedKHR"; + case DecorationOverrideCoverageNV: return "OverrideCoverageNV"; + case DecorationPassthroughNV: return "PassthroughNV"; + case DecorationViewportRelativeNV: return "ViewportRelativeNV"; + case DecorationSecondaryViewportRelativeNV: return "SecondaryViewportRelativeNV"; + case DecorationPerPrimitiveEXT: return "PerPrimitiveEXT"; + case DecorationPerViewNV: return "PerViewNV"; + case DecorationPerTaskNV: return "PerTaskNV"; + case DecorationPerVertexKHR: return "PerVertexKHR"; + case DecorationNonUniform: return "NonUniform"; + case DecorationRestrictPointer: return "RestrictPointer"; + case DecorationAliasedPointer: return "AliasedPointer"; + case DecorationMemberOffsetNV: return "MemberOffsetNV"; + case DecorationHitObjectShaderRecordBufferNV: return "HitObjectShaderRecordBufferNV"; + case DecorationHitObjectShaderRecordBufferEXT: return "HitObjectShaderRecordBufferEXT"; + case DecorationBankNV: return "BankNV"; + case DecorationBindlessSamplerNV: return "BindlessSamplerNV"; + case DecorationBindlessImageNV: return "BindlessImageNV"; + case DecorationBoundSamplerNV: return "BoundSamplerNV"; + case DecorationBoundImageNV: return "BoundImageNV"; + case DecorationSIMTCallINTEL: return "SIMTCallINTEL"; + case DecorationReferencedIndirectlyINTEL: return "ReferencedIndirectlyINTEL"; + case DecorationClobberINTEL: return "ClobberINTEL"; + case DecorationSideEffectsINTEL: return "SideEffectsINTEL"; + case DecorationVectorComputeVariableINTEL: return "VectorComputeVariableINTEL"; + case DecorationFuncParamIOKindINTEL: return "FuncParamIOKindINTEL"; + case DecorationVectorComputeFunctionINTEL: return "VectorComputeFunctionINTEL"; + case DecorationStackCallINTEL: return "StackCallINTEL"; + case DecorationGlobalVariableOffsetINTEL: return "GlobalVariableOffsetINTEL"; + case DecorationCounterBuffer: return "CounterBuffer"; + case DecorationHlslSemanticGOOGLE: return "HlslSemanticGOOGLE"; + case DecorationUserTypeGOOGLE: return "UserTypeGOOGLE"; + case DecorationFunctionRoundingModeINTEL: return "FunctionRoundingModeINTEL"; + case DecorationFunctionDenormModeINTEL: return "FunctionDenormModeINTEL"; + case DecorationRegisterALTERA: return "RegisterALTERA"; + case DecorationMemoryALTERA: return "MemoryALTERA"; + case DecorationNumbanksALTERA: return "NumbanksALTERA"; + case DecorationBankwidthALTERA: return "BankwidthALTERA"; + case DecorationMaxPrivateCopiesALTERA: return "MaxPrivateCopiesALTERA"; + case DecorationSinglepumpALTERA: return "SinglepumpALTERA"; + case DecorationDoublepumpALTERA: return "DoublepumpALTERA"; + case DecorationMaxReplicatesALTERA: return "MaxReplicatesALTERA"; + case DecorationSimpleDualPortALTERA: return "SimpleDualPortALTERA"; + case DecorationMergeALTERA: return "MergeALTERA"; + case DecorationBankBitsALTERA: return "BankBitsALTERA"; + case DecorationForcePow2DepthALTERA: return "ForcePow2DepthALTERA"; + case DecorationStridesizeALTERA: return "StridesizeALTERA"; + case DecorationWordsizeALTERA: return "WordsizeALTERA"; + case DecorationTrueDualPortALTERA: return "TrueDualPortALTERA"; + case DecorationBurstCoalesceALTERA: return "BurstCoalesceALTERA"; + case DecorationCacheSizeALTERA: return "CacheSizeALTERA"; + case DecorationDontStaticallyCoalesceALTERA: return "DontStaticallyCoalesceALTERA"; + case DecorationPrefetchALTERA: return "PrefetchALTERA"; + case DecorationStallEnableALTERA: return "StallEnableALTERA"; + case DecorationFuseLoopsInFunctionALTERA: return "FuseLoopsInFunctionALTERA"; + case DecorationMathOpDSPModeALTERA: return "MathOpDSPModeALTERA"; + case DecorationAliasScopeINTEL: return "AliasScopeINTEL"; + case DecorationNoAliasINTEL: return "NoAliasINTEL"; + case DecorationInitiationIntervalALTERA: return "InitiationIntervalALTERA"; + case DecorationMaxConcurrencyALTERA: return "MaxConcurrencyALTERA"; + case DecorationPipelineEnableALTERA: return "PipelineEnableALTERA"; + case DecorationBufferLocationALTERA: return "BufferLocationALTERA"; + case DecorationIOPipeStorageALTERA: return "IOPipeStorageALTERA"; + case DecorationFunctionFloatingPointModeINTEL: return "FunctionFloatingPointModeINTEL"; + case DecorationSingleElementVectorINTEL: return "SingleElementVectorINTEL"; + case DecorationVectorComputeCallableFunctionINTEL: return "VectorComputeCallableFunctionINTEL"; + case DecorationMediaBlockIOINTEL: return "MediaBlockIOINTEL"; + case DecorationStallFreeALTERA: return "StallFreeALTERA"; + case DecorationFPMaxErrorDecorationINTEL: return "FPMaxErrorDecorationINTEL"; + case DecorationLatencyControlLabelALTERA: return "LatencyControlLabelALTERA"; + case DecorationLatencyControlConstraintALTERA: return "LatencyControlConstraintALTERA"; + case DecorationConduitKernelArgumentALTERA: return "ConduitKernelArgumentALTERA"; + case DecorationRegisterMapKernelArgumentALTERA: return "RegisterMapKernelArgumentALTERA"; + case DecorationMMHostInterfaceAddressWidthALTERA: return "MMHostInterfaceAddressWidthALTERA"; + case DecorationMMHostInterfaceDataWidthALTERA: return "MMHostInterfaceDataWidthALTERA"; + case DecorationMMHostInterfaceLatencyALTERA: return "MMHostInterfaceLatencyALTERA"; + case DecorationMMHostInterfaceReadWriteModeALTERA: return "MMHostInterfaceReadWriteModeALTERA"; + case DecorationMMHostInterfaceMaxBurstALTERA: return "MMHostInterfaceMaxBurstALTERA"; + case DecorationMMHostInterfaceWaitRequestALTERA: return "MMHostInterfaceWaitRequestALTERA"; + case DecorationStableKernelArgumentALTERA: return "StableKernelArgumentALTERA"; + case DecorationHostAccessINTEL: return "HostAccessINTEL"; + case DecorationInitModeALTERA: return "InitModeALTERA"; + case DecorationImplementInRegisterMapALTERA: return "ImplementInRegisterMapALTERA"; + case DecorationConditionalINTEL: return "ConditionalINTEL"; + case DecorationCacheControlLoadINTEL: return "CacheControlLoadINTEL"; + case DecorationCacheControlStoreINTEL: return "CacheControlStoreINTEL"; + default: return "Unknown"; + } +} + +inline const char* BuiltInToString(BuiltIn value) { + switch (value) { + case BuiltInPosition: return "Position"; + case BuiltInPointSize: return "PointSize"; + case BuiltInClipDistance: return "ClipDistance"; + case BuiltInCullDistance: return "CullDistance"; + case BuiltInVertexId: return "VertexId"; + case BuiltInInstanceId: return "InstanceId"; + case BuiltInPrimitiveId: return "PrimitiveId"; + case BuiltInInvocationId: return "InvocationId"; + case BuiltInLayer: return "Layer"; + case BuiltInViewportIndex: return "ViewportIndex"; + case BuiltInTessLevelOuter: return "TessLevelOuter"; + case BuiltInTessLevelInner: return "TessLevelInner"; + case BuiltInTessCoord: return "TessCoord"; + case BuiltInPatchVertices: return "PatchVertices"; + case BuiltInFragCoord: return "FragCoord"; + case BuiltInPointCoord: return "PointCoord"; + case BuiltInFrontFacing: return "FrontFacing"; + case BuiltInSampleId: return "SampleId"; + case BuiltInSamplePosition: return "SamplePosition"; + case BuiltInSampleMask: return "SampleMask"; + case BuiltInFragDepth: return "FragDepth"; + case BuiltInHelperInvocation: return "HelperInvocation"; + case BuiltInNumWorkgroups: return "NumWorkgroups"; + case BuiltInWorkgroupSize: return "WorkgroupSize"; + case BuiltInWorkgroupId: return "WorkgroupId"; + case BuiltInLocalInvocationId: return "LocalInvocationId"; + case BuiltInGlobalInvocationId: return "GlobalInvocationId"; + case BuiltInLocalInvocationIndex: return "LocalInvocationIndex"; + case BuiltInWorkDim: return "WorkDim"; + case BuiltInGlobalSize: return "GlobalSize"; + case BuiltInEnqueuedWorkgroupSize: return "EnqueuedWorkgroupSize"; + case BuiltInGlobalOffset: return "GlobalOffset"; + case BuiltInGlobalLinearId: return "GlobalLinearId"; + case BuiltInSubgroupSize: return "SubgroupSize"; + case BuiltInSubgroupMaxSize: return "SubgroupMaxSize"; + case BuiltInNumSubgroups: return "NumSubgroups"; + case BuiltInNumEnqueuedSubgroups: return "NumEnqueuedSubgroups"; + case BuiltInSubgroupId: return "SubgroupId"; + case BuiltInSubgroupLocalInvocationId: return "SubgroupLocalInvocationId"; + case BuiltInVertexIndex: return "VertexIndex"; + case BuiltInInstanceIndex: return "InstanceIndex"; + case BuiltInCoreIDARM: return "CoreIDARM"; + case BuiltInCoreCountARM: return "CoreCountARM"; + case BuiltInCoreMaxIDARM: return "CoreMaxIDARM"; + case BuiltInWarpIDARM: return "WarpIDARM"; + case BuiltInWarpMaxIDARM: return "WarpMaxIDARM"; + case BuiltInSubgroupEqMask: return "SubgroupEqMask"; + case BuiltInSubgroupGeMask: return "SubgroupGeMask"; + case BuiltInSubgroupGtMask: return "SubgroupGtMask"; + case BuiltInSubgroupLeMask: return "SubgroupLeMask"; + case BuiltInSubgroupLtMask: return "SubgroupLtMask"; + case BuiltInBaseVertex: return "BaseVertex"; + case BuiltInBaseInstance: return "BaseInstance"; + case BuiltInDrawIndex: return "DrawIndex"; + case BuiltInPrimitiveShadingRateKHR: return "PrimitiveShadingRateKHR"; + case BuiltInDeviceIndex: return "DeviceIndex"; + case BuiltInViewIndex: return "ViewIndex"; + case BuiltInShadingRateKHR: return "ShadingRateKHR"; + case BuiltInTileOffsetQCOM: return "TileOffsetQCOM"; + case BuiltInTileDimensionQCOM: return "TileDimensionQCOM"; + case BuiltInTileApronSizeQCOM: return "TileApronSizeQCOM"; + case BuiltInBaryCoordNoPerspAMD: return "BaryCoordNoPerspAMD"; + case BuiltInBaryCoordNoPerspCentroidAMD: return "BaryCoordNoPerspCentroidAMD"; + case BuiltInBaryCoordNoPerspSampleAMD: return "BaryCoordNoPerspSampleAMD"; + case BuiltInBaryCoordSmoothAMD: return "BaryCoordSmoothAMD"; + case BuiltInBaryCoordSmoothCentroidAMD: return "BaryCoordSmoothCentroidAMD"; + case BuiltInBaryCoordSmoothSampleAMD: return "BaryCoordSmoothSampleAMD"; + case BuiltInBaryCoordPullModelAMD: return "BaryCoordPullModelAMD"; + case BuiltInFragStencilRefEXT: return "FragStencilRefEXT"; + case BuiltInRemainingRecursionLevelsAMDX: return "RemainingRecursionLevelsAMDX"; + case BuiltInShaderIndexAMDX: return "ShaderIndexAMDX"; + case BuiltInSamplerHeapEXT: return "SamplerHeapEXT"; + case BuiltInResourceHeapEXT: return "ResourceHeapEXT"; + case BuiltInViewportMaskNV: return "ViewportMaskNV"; + case BuiltInSecondaryPositionNV: return "SecondaryPositionNV"; + case BuiltInSecondaryViewportMaskNV: return "SecondaryViewportMaskNV"; + case BuiltInPositionPerViewNV: return "PositionPerViewNV"; + case BuiltInViewportMaskPerViewNV: return "ViewportMaskPerViewNV"; + case BuiltInFullyCoveredEXT: return "FullyCoveredEXT"; + case BuiltInTaskCountNV: return "TaskCountNV"; + case BuiltInPrimitiveCountNV: return "PrimitiveCountNV"; + case BuiltInPrimitiveIndicesNV: return "PrimitiveIndicesNV"; + case BuiltInClipDistancePerViewNV: return "ClipDistancePerViewNV"; + case BuiltInCullDistancePerViewNV: return "CullDistancePerViewNV"; + case BuiltInLayerPerViewNV: return "LayerPerViewNV"; + case BuiltInMeshViewCountNV: return "MeshViewCountNV"; + case BuiltInMeshViewIndicesNV: return "MeshViewIndicesNV"; + case BuiltInBaryCoordKHR: return "BaryCoordKHR"; + case BuiltInBaryCoordNoPerspKHR: return "BaryCoordNoPerspKHR"; + case BuiltInFragSizeEXT: return "FragSizeEXT"; + case BuiltInFragInvocationCountEXT: return "FragInvocationCountEXT"; + case BuiltInPrimitivePointIndicesEXT: return "PrimitivePointIndicesEXT"; + case BuiltInPrimitiveLineIndicesEXT: return "PrimitiveLineIndicesEXT"; + case BuiltInPrimitiveTriangleIndicesEXT: return "PrimitiveTriangleIndicesEXT"; + case BuiltInCullPrimitiveEXT: return "CullPrimitiveEXT"; + case BuiltInLaunchIdKHR: return "LaunchIdKHR"; + case BuiltInLaunchSizeKHR: return "LaunchSizeKHR"; + case BuiltInWorldRayOriginKHR: return "WorldRayOriginKHR"; + case BuiltInWorldRayDirectionKHR: return "WorldRayDirectionKHR"; + case BuiltInObjectRayOriginKHR: return "ObjectRayOriginKHR"; + case BuiltInObjectRayDirectionKHR: return "ObjectRayDirectionKHR"; + case BuiltInRayTminKHR: return "RayTminKHR"; + case BuiltInRayTmaxKHR: return "RayTmaxKHR"; + case BuiltInInstanceCustomIndexKHR: return "InstanceCustomIndexKHR"; + case BuiltInObjectToWorldKHR: return "ObjectToWorldKHR"; + case BuiltInWorldToObjectKHR: return "WorldToObjectKHR"; + case BuiltInHitTNV: return "HitTNV"; + case BuiltInHitKindKHR: return "HitKindKHR"; + case BuiltInCurrentRayTimeNV: return "CurrentRayTimeNV"; + case BuiltInHitTriangleVertexPositionsKHR: return "HitTriangleVertexPositionsKHR"; + case BuiltInHitMicroTriangleVertexPositionsNV: return "HitMicroTriangleVertexPositionsNV"; + case BuiltInHitMicroTriangleVertexBarycentricsNV: return "HitMicroTriangleVertexBarycentricsNV"; + case BuiltInIncomingRayFlagsKHR: return "IncomingRayFlagsKHR"; + case BuiltInRayGeometryIndexKHR: return "RayGeometryIndexKHR"; + case BuiltInHitIsSphereNV: return "HitIsSphereNV"; + case BuiltInHitIsLSSNV: return "HitIsLSSNV"; + case BuiltInHitSpherePositionNV: return "HitSpherePositionNV"; + case BuiltInWarpsPerSMNV: return "WarpsPerSMNV"; + case BuiltInSMCountNV: return "SMCountNV"; + case BuiltInWarpIDNV: return "WarpIDNV"; + case BuiltInSMIDNV: return "SMIDNV"; + case BuiltInHitLSSPositionsNV: return "HitLSSPositionsNV"; + case BuiltInHitKindFrontFacingMicroTriangleNV: return "HitKindFrontFacingMicroTriangleNV"; + case BuiltInHitKindBackFacingMicroTriangleNV: return "HitKindBackFacingMicroTriangleNV"; + case BuiltInHitSphereRadiusNV: return "HitSphereRadiusNV"; + case BuiltInHitLSSRadiiNV: return "HitLSSRadiiNV"; + case BuiltInClusterIDNV: return "ClusterIDNV"; + case BuiltInCullMaskKHR: return "CullMaskKHR"; + default: return "Unknown"; + } +} + +inline const char* ScopeToString(Scope value) { + switch (value) { + case ScopeCrossDevice: return "CrossDevice"; + case ScopeDevice: return "Device"; + case ScopeWorkgroup: return "Workgroup"; + case ScopeSubgroup: return "Subgroup"; + case ScopeInvocation: return "Invocation"; + case ScopeQueueFamily: return "QueueFamily"; + case ScopeShaderCallKHR: return "ShaderCallKHR"; + default: return "Unknown"; + } +} + +inline const char* GroupOperationToString(GroupOperation value) { + switch (value) { + case GroupOperationReduce: return "Reduce"; + case GroupOperationInclusiveScan: return "InclusiveScan"; + case GroupOperationExclusiveScan: return "ExclusiveScan"; + case GroupOperationClusteredReduce: return "ClusteredReduce"; + case GroupOperationPartitionedReduceEXT: return "PartitionedReduceEXT"; + case GroupOperationPartitionedInclusiveScanEXT: return "PartitionedInclusiveScanEXT"; + case GroupOperationPartitionedExclusiveScanEXT: return "PartitionedExclusiveScanEXT"; + default: return "Unknown"; + } +} + +inline const char* KernelEnqueueFlagsToString(KernelEnqueueFlags value) { + switch (value) { + case KernelEnqueueFlagsNoWait: return "NoWait"; + case KernelEnqueueFlagsWaitKernel: return "WaitKernel"; + case KernelEnqueueFlagsWaitWorkGroup: return "WaitWorkGroup"; + default: return "Unknown"; + } +} + +inline const char* CapabilityToString(Capability value) { + switch (value) { + case CapabilityMatrix: return "Matrix"; + case CapabilityShader: return "Shader"; + case CapabilityGeometry: return "Geometry"; + case CapabilityTessellation: return "Tessellation"; + case CapabilityAddresses: return "Addresses"; + case CapabilityLinkage: return "Linkage"; + case CapabilityKernel: return "Kernel"; + case CapabilityVector16: return "Vector16"; + case CapabilityFloat16Buffer: return "Float16Buffer"; + case CapabilityFloat16: return "Float16"; + case CapabilityFloat64: return "Float64"; + case CapabilityInt64: return "Int64"; + case CapabilityInt64Atomics: return "Int64Atomics"; + case CapabilityImageBasic: return "ImageBasic"; + case CapabilityImageReadWrite: return "ImageReadWrite"; + case CapabilityImageMipmap: return "ImageMipmap"; + case CapabilityPipes: return "Pipes"; + case CapabilityGroups: return "Groups"; + case CapabilityDeviceEnqueue: return "DeviceEnqueue"; + case CapabilityLiteralSampler: return "LiteralSampler"; + case CapabilityAtomicStorage: return "AtomicStorage"; + case CapabilityInt16: return "Int16"; + case CapabilityTessellationPointSize: return "TessellationPointSize"; + case CapabilityGeometryPointSize: return "GeometryPointSize"; + case CapabilityImageGatherExtended: return "ImageGatherExtended"; + case CapabilityStorageImageMultisample: return "StorageImageMultisample"; + case CapabilityUniformBufferArrayDynamicIndexing: return "UniformBufferArrayDynamicIndexing"; + case CapabilitySampledImageArrayDynamicIndexing: return "SampledImageArrayDynamicIndexing"; + case CapabilityStorageBufferArrayDynamicIndexing: return "StorageBufferArrayDynamicIndexing"; + case CapabilityStorageImageArrayDynamicIndexing: return "StorageImageArrayDynamicIndexing"; + case CapabilityClipDistance: return "ClipDistance"; + case CapabilityCullDistance: return "CullDistance"; + case CapabilityImageCubeArray: return "ImageCubeArray"; + case CapabilitySampleRateShading: return "SampleRateShading"; + case CapabilityImageRect: return "ImageRect"; + case CapabilitySampledRect: return "SampledRect"; + case CapabilityGenericPointer: return "GenericPointer"; + case CapabilityInt8: return "Int8"; + case CapabilityInputAttachment: return "InputAttachment"; + case CapabilitySparseResidency: return "SparseResidency"; + case CapabilityMinLod: return "MinLod"; + case CapabilitySampled1D: return "Sampled1D"; + case CapabilityImage1D: return "Image1D"; + case CapabilitySampledCubeArray: return "SampledCubeArray"; + case CapabilitySampledBuffer: return "SampledBuffer"; + case CapabilityImageBuffer: return "ImageBuffer"; + case CapabilityImageMSArray: return "ImageMSArray"; + case CapabilityStorageImageExtendedFormats: return "StorageImageExtendedFormats"; + case CapabilityImageQuery: return "ImageQuery"; + case CapabilityDerivativeControl: return "DerivativeControl"; + case CapabilityInterpolationFunction: return "InterpolationFunction"; + case CapabilityTransformFeedback: return "TransformFeedback"; + case CapabilityGeometryStreams: return "GeometryStreams"; + case CapabilityStorageImageReadWithoutFormat: return "StorageImageReadWithoutFormat"; + case CapabilityStorageImageWriteWithoutFormat: return "StorageImageWriteWithoutFormat"; + case CapabilityMultiViewport: return "MultiViewport"; + case CapabilitySubgroupDispatch: return "SubgroupDispatch"; + case CapabilityNamedBarrier: return "NamedBarrier"; + case CapabilityPipeStorage: return "PipeStorage"; + case CapabilityGroupNonUniform: return "GroupNonUniform"; + case CapabilityGroupNonUniformVote: return "GroupNonUniformVote"; + case CapabilityGroupNonUniformArithmetic: return "GroupNonUniformArithmetic"; + case CapabilityGroupNonUniformBallot: return "GroupNonUniformBallot"; + case CapabilityGroupNonUniformShuffle: return "GroupNonUniformShuffle"; + case CapabilityGroupNonUniformShuffleRelative: return "GroupNonUniformShuffleRelative"; + case CapabilityGroupNonUniformClustered: return "GroupNonUniformClustered"; + case CapabilityGroupNonUniformQuad: return "GroupNonUniformQuad"; + case CapabilityShaderLayer: return "ShaderLayer"; + case CapabilityShaderViewportIndex: return "ShaderViewportIndex"; + case CapabilityUniformDecoration: return "UniformDecoration"; + case CapabilityCoreBuiltinsARM: return "CoreBuiltinsARM"; + case CapabilityTileImageColorReadAccessEXT: return "TileImageColorReadAccessEXT"; + case CapabilityTileImageDepthReadAccessEXT: return "TileImageDepthReadAccessEXT"; + case CapabilityTileImageStencilReadAccessEXT: return "TileImageStencilReadAccessEXT"; + case CapabilityTensorsARM: return "TensorsARM"; + case CapabilityStorageTensorArrayDynamicIndexingARM: return "StorageTensorArrayDynamicIndexingARM"; + case CapabilityStorageTensorArrayNonUniformIndexingARM: return "StorageTensorArrayNonUniformIndexingARM"; + case CapabilityGraphARM: return "GraphARM"; + case CapabilityCooperativeMatrixLayoutsARM: return "CooperativeMatrixLayoutsARM"; + case CapabilityFloat8EXT: return "Float8EXT"; + case CapabilityFloat8CooperativeMatrixEXT: return "Float8CooperativeMatrixEXT"; + case CapabilityFragmentShadingRateKHR: return "FragmentShadingRateKHR"; + case CapabilitySubgroupBallotKHR: return "SubgroupBallotKHR"; + case CapabilityDrawParameters: return "DrawParameters"; + case CapabilityWorkgroupMemoryExplicitLayoutKHR: return "WorkgroupMemoryExplicitLayoutKHR"; + case CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR: return "WorkgroupMemoryExplicitLayout8BitAccessKHR"; + case CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR: return "WorkgroupMemoryExplicitLayout16BitAccessKHR"; + case CapabilitySubgroupVoteKHR: return "SubgroupVoteKHR"; + case CapabilityStorageBuffer16BitAccess: return "StorageBuffer16BitAccess"; + case CapabilityStorageUniform16: return "StorageUniform16"; + case CapabilityStoragePushConstant16: return "StoragePushConstant16"; + case CapabilityStorageInputOutput16: return "StorageInputOutput16"; + case CapabilityDeviceGroup: return "DeviceGroup"; + case CapabilityMultiView: return "MultiView"; + case CapabilityVariablePointersStorageBuffer: return "VariablePointersStorageBuffer"; + case CapabilityVariablePointers: return "VariablePointers"; + case CapabilityAtomicStorageOps: return "AtomicStorageOps"; + case CapabilitySampleMaskPostDepthCoverage: return "SampleMaskPostDepthCoverage"; + case CapabilityStorageBuffer8BitAccess: return "StorageBuffer8BitAccess"; + case CapabilityUniformAndStorageBuffer8BitAccess: return "UniformAndStorageBuffer8BitAccess"; + case CapabilityStoragePushConstant8: return "StoragePushConstant8"; + case CapabilityDenormPreserve: return "DenormPreserve"; + case CapabilityDenormFlushToZero: return "DenormFlushToZero"; + case CapabilitySignedZeroInfNanPreserve: return "SignedZeroInfNanPreserve"; + case CapabilityRoundingModeRTE: return "RoundingModeRTE"; + case CapabilityRoundingModeRTZ: return "RoundingModeRTZ"; + case CapabilityRayQueryProvisionalKHR: return "RayQueryProvisionalKHR"; + case CapabilityRayQueryKHR: return "RayQueryKHR"; + case CapabilityUntypedPointersKHR: return "UntypedPointersKHR"; + case CapabilityRayTraversalPrimitiveCullingKHR: return "RayTraversalPrimitiveCullingKHR"; + case CapabilityRayTracingKHR: return "RayTracingKHR"; + case CapabilityTextureSampleWeightedQCOM: return "TextureSampleWeightedQCOM"; + case CapabilityTextureBoxFilterQCOM: return "TextureBoxFilterQCOM"; + case CapabilityTextureBlockMatchQCOM: return "TextureBlockMatchQCOM"; + case CapabilityTileShadingQCOM: return "TileShadingQCOM"; + case CapabilityCooperativeMatrixConversionQCOM: return "CooperativeMatrixConversionQCOM"; + case CapabilityTextureBlockMatch2QCOM: return "TextureBlockMatch2QCOM"; + case CapabilityFloat16ImageAMD: return "Float16ImageAMD"; + case CapabilityImageGatherBiasLodAMD: return "ImageGatherBiasLodAMD"; + case CapabilityFragmentMaskAMD: return "FragmentMaskAMD"; + case CapabilityStencilExportEXT: return "StencilExportEXT"; + case CapabilityImageReadWriteLodAMD: return "ImageReadWriteLodAMD"; + case CapabilityInt64ImageEXT: return "Int64ImageEXT"; + case CapabilityShaderClockKHR: return "ShaderClockKHR"; + case CapabilityShaderEnqueueAMDX: return "ShaderEnqueueAMDX"; + case CapabilityQuadControlKHR: return "QuadControlKHR"; + case CapabilityInt4TypeINTEL: return "Int4TypeINTEL"; + case CapabilityInt4CooperativeMatrixINTEL: return "Int4CooperativeMatrixINTEL"; + case CapabilityBFloat16TypeKHR: return "BFloat16TypeKHR"; + case CapabilityBFloat16DotProductKHR: return "BFloat16DotProductKHR"; + case CapabilityBFloat16CooperativeMatrixKHR: return "BFloat16CooperativeMatrixKHR"; + case CapabilityAbortKHR: return "AbortKHR"; + case CapabilityDescriptorHeapEXT: return "DescriptorHeapEXT"; + case CapabilityConstantDataKHR: return "ConstantDataKHR"; + case CapabilityPoisonFreezeKHR: return "PoisonFreezeKHR"; + case CapabilitySampleMaskOverrideCoverageNV: return "SampleMaskOverrideCoverageNV"; + case CapabilityGeometryShaderPassthroughNV: return "GeometryShaderPassthroughNV"; + case CapabilityShaderViewportIndexLayerEXT: return "ShaderViewportIndexLayerEXT"; + case CapabilityShaderViewportMaskNV: return "ShaderViewportMaskNV"; + case CapabilityShaderStereoViewNV: return "ShaderStereoViewNV"; + case CapabilityPerViewAttributesNV: return "PerViewAttributesNV"; + case CapabilityFragmentFullyCoveredEXT: return "FragmentFullyCoveredEXT"; + case CapabilityMeshShadingNV: return "MeshShadingNV"; + case CapabilityImageFootprintNV: return "ImageFootprintNV"; + case CapabilityMeshShadingEXT: return "MeshShadingEXT"; + case CapabilityFragmentBarycentricKHR: return "FragmentBarycentricKHR"; + case CapabilityComputeDerivativeGroupQuadsKHR: return "ComputeDerivativeGroupQuadsKHR"; + case CapabilityFragmentDensityEXT: return "FragmentDensityEXT"; + case CapabilityGroupNonUniformPartitionedEXT: return "GroupNonUniformPartitionedEXT"; + case CapabilityShaderNonUniform: return "ShaderNonUniform"; + case CapabilityRuntimeDescriptorArray: return "RuntimeDescriptorArray"; + case CapabilityInputAttachmentArrayDynamicIndexing: return "InputAttachmentArrayDynamicIndexing"; + case CapabilityUniformTexelBufferArrayDynamicIndexing: return "UniformTexelBufferArrayDynamicIndexing"; + case CapabilityStorageTexelBufferArrayDynamicIndexing: return "StorageTexelBufferArrayDynamicIndexing"; + case CapabilityUniformBufferArrayNonUniformIndexing: return "UniformBufferArrayNonUniformIndexing"; + case CapabilitySampledImageArrayNonUniformIndexing: return "SampledImageArrayNonUniformIndexing"; + case CapabilityStorageBufferArrayNonUniformIndexing: return "StorageBufferArrayNonUniformIndexing"; + case CapabilityStorageImageArrayNonUniformIndexing: return "StorageImageArrayNonUniformIndexing"; + case CapabilityInputAttachmentArrayNonUniformIndexing: return "InputAttachmentArrayNonUniformIndexing"; + case CapabilityUniformTexelBufferArrayNonUniformIndexing: return "UniformTexelBufferArrayNonUniformIndexing"; + case CapabilityStorageTexelBufferArrayNonUniformIndexing: return "StorageTexelBufferArrayNonUniformIndexing"; + case CapabilityRayTracingPositionFetchKHR: return "RayTracingPositionFetchKHR"; + case CapabilityRayTracingNV: return "RayTracingNV"; + case CapabilityRayTracingMotionBlurNV: return "RayTracingMotionBlurNV"; + case CapabilityVulkanMemoryModel: return "VulkanMemoryModel"; + case CapabilityVulkanMemoryModelDeviceScope: return "VulkanMemoryModelDeviceScope"; + case CapabilityPhysicalStorageBufferAddresses: return "PhysicalStorageBufferAddresses"; + case CapabilityComputeDerivativeGroupLinearKHR: return "ComputeDerivativeGroupLinearKHR"; + case CapabilityRayTracingProvisionalKHR: return "RayTracingProvisionalKHR"; + case CapabilityCooperativeMatrixNV: return "CooperativeMatrixNV"; + case CapabilityFragmentShaderSampleInterlockEXT: return "FragmentShaderSampleInterlockEXT"; + case CapabilityFragmentShaderShadingRateInterlockEXT: return "FragmentShaderShadingRateInterlockEXT"; + case CapabilityShaderSMBuiltinsNV: return "ShaderSMBuiltinsNV"; + case CapabilityFragmentShaderPixelInterlockEXT: return "FragmentShaderPixelInterlockEXT"; + case CapabilityDemoteToHelperInvocation: return "DemoteToHelperInvocation"; + case CapabilityDisplacementMicromapNV: return "DisplacementMicromapNV"; + case CapabilityRayTracingOpacityMicromapEXT: return "RayTracingOpacityMicromapEXT"; + case CapabilityShaderInvocationReorderNV: return "ShaderInvocationReorderNV"; + case CapabilityShaderInvocationReorderEXT: return "ShaderInvocationReorderEXT"; + case CapabilityBindlessTextureNV: return "BindlessTextureNV"; + case CapabilityRayQueryPositionFetchKHR: return "RayQueryPositionFetchKHR"; + case CapabilityCooperativeVectorNV: return "CooperativeVectorNV"; + case CapabilityAtomicFloat16VectorNV: return "AtomicFloat16VectorNV"; + case CapabilityRayTracingDisplacementMicromapNV: return "RayTracingDisplacementMicromapNV"; + case CapabilityRawAccessChainsNV: return "RawAccessChainsNV"; + case CapabilityRayTracingSpheresGeometryNV: return "RayTracingSpheresGeometryNV"; + case CapabilityRayTracingLinearSweptSpheresGeometryNV: return "RayTracingLinearSweptSpheresGeometryNV"; + case CapabilityPushConstantBanksNV: return "PushConstantBanksNV"; + case CapabilityLongVectorEXT: return "LongVectorEXT"; + case CapabilityShader64BitIndexingEXT: return "Shader64BitIndexingEXT"; + case CapabilityCooperativeMatrixReductionsNV: return "CooperativeMatrixReductionsNV"; + case CapabilityCooperativeMatrixConversionsNV: return "CooperativeMatrixConversionsNV"; + case CapabilityCooperativeMatrixPerElementOperationsNV: return "CooperativeMatrixPerElementOperationsNV"; + case CapabilityCooperativeMatrixTensorAddressingNV: return "CooperativeMatrixTensorAddressingNV"; + case CapabilityCooperativeMatrixBlockLoadsNV: return "CooperativeMatrixBlockLoadsNV"; + case CapabilityCooperativeVectorTrainingNV: return "CooperativeVectorTrainingNV"; + case CapabilityRayTracingClusterAccelerationStructureNV: return "RayTracingClusterAccelerationStructureNV"; + case CapabilityTensorAddressingNV: return "TensorAddressingNV"; + case CapabilitySubgroupShuffleINTEL: return "SubgroupShuffleINTEL"; + case CapabilitySubgroupBufferBlockIOINTEL: return "SubgroupBufferBlockIOINTEL"; + case CapabilitySubgroupImageBlockIOINTEL: return "SubgroupImageBlockIOINTEL"; + case CapabilitySubgroupImageMediaBlockIOINTEL: return "SubgroupImageMediaBlockIOINTEL"; + case CapabilityRoundToInfinityINTEL: return "RoundToInfinityINTEL"; + case CapabilityFloatingPointModeINTEL: return "FloatingPointModeINTEL"; + case CapabilityIntegerFunctions2INTEL: return "IntegerFunctions2INTEL"; + case CapabilityFunctionPointersINTEL: return "FunctionPointersINTEL"; + case CapabilityIndirectReferencesINTEL: return "IndirectReferencesINTEL"; + case CapabilityAsmINTEL: return "AsmINTEL"; + case CapabilityAtomicFloat32MinMaxEXT: return "AtomicFloat32MinMaxEXT"; + case CapabilityAtomicFloat64MinMaxEXT: return "AtomicFloat64MinMaxEXT"; + case CapabilityAtomicFloat16MinMaxEXT: return "AtomicFloat16MinMaxEXT"; + case CapabilityVectorComputeINTEL: return "VectorComputeINTEL"; + case CapabilityVectorAnyINTEL: return "VectorAnyINTEL"; + case CapabilityExpectAssumeKHR: return "ExpectAssumeKHR"; + case CapabilitySubgroupAvcMotionEstimationINTEL: return "SubgroupAvcMotionEstimationINTEL"; + case CapabilitySubgroupAvcMotionEstimationIntraINTEL: return "SubgroupAvcMotionEstimationIntraINTEL"; + case CapabilitySubgroupAvcMotionEstimationChromaINTEL: return "SubgroupAvcMotionEstimationChromaINTEL"; + case CapabilityVariableLengthArrayINTEL: return "VariableLengthArrayINTEL"; + case CapabilityFunctionFloatControlINTEL: return "FunctionFloatControlINTEL"; + case CapabilityFPGAMemoryAttributesALTERA: return "FPGAMemoryAttributesALTERA"; + case CapabilityFPFastMathModeINTEL: return "FPFastMathModeINTEL"; + case CapabilityArbitraryPrecisionIntegersALTERA: return "ArbitraryPrecisionIntegersALTERA"; + case CapabilityArbitraryPrecisionFloatingPointALTERA: return "ArbitraryPrecisionFloatingPointALTERA"; + case CapabilityUnstructuredLoopControlsINTEL: return "UnstructuredLoopControlsINTEL"; + case CapabilityFPGALoopControlsALTERA: return "FPGALoopControlsALTERA"; + case CapabilityKernelAttributesINTEL: return "KernelAttributesINTEL"; + case CapabilityFPGAKernelAttributesINTEL: return "FPGAKernelAttributesINTEL"; + case CapabilityFPGAMemoryAccessesALTERA: return "FPGAMemoryAccessesALTERA"; + case CapabilityFPGAClusterAttributesALTERA: return "FPGAClusterAttributesALTERA"; + case CapabilityLoopFuseALTERA: return "LoopFuseALTERA"; + case CapabilityFPGADSPControlALTERA: return "FPGADSPControlALTERA"; + case CapabilityMemoryAccessAliasingINTEL: return "MemoryAccessAliasingINTEL"; + case CapabilityFPGAInvocationPipeliningAttributesALTERA: return "FPGAInvocationPipeliningAttributesALTERA"; + case CapabilityFPGABufferLocationALTERA: return "FPGABufferLocationALTERA"; + case CapabilityArbitraryPrecisionFixedPointALTERA: return "ArbitraryPrecisionFixedPointALTERA"; + case CapabilityUSMStorageClassesALTERA: return "USMStorageClassesALTERA"; + case CapabilityRuntimeAlignedAttributeALTERA: return "RuntimeAlignedAttributeALTERA"; + case CapabilityIOPipesALTERA: return "IOPipesALTERA"; + case CapabilityBlockingPipesALTERA: return "BlockingPipesALTERA"; + case CapabilityFPGARegALTERA: return "FPGARegALTERA"; + case CapabilityDotProductInputAll: return "DotProductInputAll"; + case CapabilityDotProductInput4x8Bit: return "DotProductInput4x8Bit"; + case CapabilityDotProductInput4x8BitPacked: return "DotProductInput4x8BitPacked"; + case CapabilityDotProduct: return "DotProduct"; + case CapabilityRayCullMaskKHR: return "RayCullMaskKHR"; + case CapabilityCooperativeMatrixKHR: return "CooperativeMatrixKHR"; + case CapabilityReplicatedCompositesEXT: return "ReplicatedCompositesEXT"; + case CapabilityBitInstructions: return "BitInstructions"; + case CapabilityGroupNonUniformRotateKHR: return "GroupNonUniformRotateKHR"; + case CapabilityFloatControls2: return "FloatControls2"; + case CapabilityFMAKHR: return "FMAKHR"; + case CapabilityAtomicFloat32AddEXT: return "AtomicFloat32AddEXT"; + case CapabilityAtomicFloat64AddEXT: return "AtomicFloat64AddEXT"; + case CapabilityLongCompositesINTEL: return "LongCompositesINTEL"; + case CapabilityOptNoneEXT: return "OptNoneEXT"; + case CapabilityAtomicFloat16AddEXT: return "AtomicFloat16AddEXT"; + case CapabilityDebugInfoModuleINTEL: return "DebugInfoModuleINTEL"; + case CapabilityBFloat16ConversionINTEL: return "BFloat16ConversionINTEL"; + case CapabilitySplitBarrierINTEL: return "SplitBarrierINTEL"; + case CapabilityArithmeticFenceEXT: return "ArithmeticFenceEXT"; + case CapabilityFPGAClusterAttributesV2ALTERA: return "FPGAClusterAttributesV2ALTERA"; + case CapabilityFPGAKernelAttributesv2INTEL: return "FPGAKernelAttributesv2INTEL"; + case CapabilityTaskSequenceALTERA: return "TaskSequenceALTERA"; + case CapabilityFPMaxErrorINTEL: return "FPMaxErrorINTEL"; + case CapabilityFPGALatencyControlALTERA: return "FPGALatencyControlALTERA"; + case CapabilityFPGAArgumentInterfacesALTERA: return "FPGAArgumentInterfacesALTERA"; + case CapabilityGlobalVariableHostAccessINTEL: return "GlobalVariableHostAccessINTEL"; + case CapabilityGlobalVariableFPGADecorationsALTERA: return "GlobalVariableFPGADecorationsALTERA"; + case CapabilitySubgroupBufferPrefetchINTEL: return "SubgroupBufferPrefetchINTEL"; + case CapabilitySubgroup2DBlockIOINTEL: return "Subgroup2DBlockIOINTEL"; + case CapabilitySubgroup2DBlockTransformINTEL: return "Subgroup2DBlockTransformINTEL"; + case CapabilitySubgroup2DBlockTransposeINTEL: return "Subgroup2DBlockTransposeINTEL"; + case CapabilitySubgroupMatrixMultiplyAccumulateINTEL: return "SubgroupMatrixMultiplyAccumulateINTEL"; + case CapabilityTernaryBitwiseFunctionINTEL: return "TernaryBitwiseFunctionINTEL"; + case CapabilityUntypedVariableLengthArrayINTEL: return "UntypedVariableLengthArrayINTEL"; + case CapabilitySpecConditionalINTEL: return "SpecConditionalINTEL"; + case CapabilityFunctionVariantsINTEL: return "FunctionVariantsINTEL"; + case CapabilityGroupUniformArithmeticKHR: return "GroupUniformArithmeticKHR"; + case CapabilityTensorFloat32RoundingINTEL: return "TensorFloat32RoundingINTEL"; + case CapabilityMaskedGatherScatterINTEL: return "MaskedGatherScatterINTEL"; + case CapabilityCacheControlsINTEL: return "CacheControlsINTEL"; + case CapabilityRegisterLimitsINTEL: return "RegisterLimitsINTEL"; + case CapabilityBindlessImagesINTEL: return "BindlessImagesINTEL"; + case CapabilityDotProductFloat16AccFloat32VALVE: return "DotProductFloat16AccFloat32VALVE"; + case CapabilityDotProductFloat16AccFloat16VALVE: return "DotProductFloat16AccFloat16VALVE"; + case CapabilityDotProductBFloat16AccVALVE: return "DotProductBFloat16AccVALVE"; + case CapabilityDotProductFloat8AccFloat32VALVE: return "DotProductFloat8AccFloat32VALVE"; + default: return "Unknown"; + } +} + +inline const char* RayQueryIntersectionToString(RayQueryIntersection value) { + switch (value) { + case RayQueryIntersectionRayQueryCandidateIntersectionKHR: return "RayQueryCandidateIntersectionKHR"; + case RayQueryIntersectionRayQueryCommittedIntersectionKHR: return "RayQueryCommittedIntersectionKHR"; + default: return "Unknown"; + } +} + +inline const char* RayQueryCommittedIntersectionTypeToString(RayQueryCommittedIntersectionType value) { + switch (value) { + case RayQueryCommittedIntersectionTypeRayQueryCommittedIntersectionNoneKHR: return "RayQueryCommittedIntersectionNoneKHR"; + case RayQueryCommittedIntersectionTypeRayQueryCommittedIntersectionTriangleKHR: return "RayQueryCommittedIntersectionTriangleKHR"; + case RayQueryCommittedIntersectionTypeRayQueryCommittedIntersectionGeneratedKHR: return "RayQueryCommittedIntersectionGeneratedKHR"; + default: return "Unknown"; + } +} + +inline const char* RayQueryCandidateIntersectionTypeToString(RayQueryCandidateIntersectionType value) { + switch (value) { + case RayQueryCandidateIntersectionTypeRayQueryCandidateIntersectionTriangleKHR: return "RayQueryCandidateIntersectionTriangleKHR"; + case RayQueryCandidateIntersectionTypeRayQueryCandidateIntersectionAABBKHR: return "RayQueryCandidateIntersectionAABBKHR"; + default: return "Unknown"; + } +} + +inline const char* FPDenormModeToString(FPDenormMode value) { + switch (value) { + case FPDenormModePreserve: return "Preserve"; + case FPDenormModeFlushToZero: return "FlushToZero"; + default: return "Unknown"; + } +} + +inline const char* FPOperationModeToString(FPOperationMode value) { + switch (value) { + case FPOperationModeIEEE: return "IEEE"; + case FPOperationModeALT: return "ALT"; + default: return "Unknown"; + } +} + +inline const char* QuantizationModesToString(QuantizationModes value) { + switch (value) { + case QuantizationModesTRN: return "TRN"; + case QuantizationModesTRN_ZERO: return "TRN_ZERO"; + case QuantizationModesRND: return "RND"; + case QuantizationModesRND_ZERO: return "RND_ZERO"; + case QuantizationModesRND_INF: return "RND_INF"; + case QuantizationModesRND_MIN_INF: return "RND_MIN_INF"; + case QuantizationModesRND_CONV: return "RND_CONV"; + case QuantizationModesRND_CONV_ODD: return "RND_CONV_ODD"; + default: return "Unknown"; + } +} + +inline const char* OverflowModesToString(OverflowModes value) { + switch (value) { + case OverflowModesWRAP: return "WRAP"; + case OverflowModesSAT: return "SAT"; + case OverflowModesSAT_ZERO: return "SAT_ZERO"; + case OverflowModesSAT_SYM: return "SAT_SYM"; + default: return "Unknown"; + } +} + +inline const char* PackedVectorFormatToString(PackedVectorFormat value) { + switch (value) { + case PackedVectorFormatPackedVectorFormat4x8Bit: return "PackedVectorFormat4x8Bit"; + default: return "Unknown"; + } +} + +inline const char* CooperativeMatrixLayoutToString(CooperativeMatrixLayout value) { + switch (value) { + case CooperativeMatrixLayoutRowMajorKHR: return "RowMajorKHR"; + case CooperativeMatrixLayoutColumnMajorKHR: return "ColumnMajorKHR"; + case CooperativeMatrixLayoutRowBlockedInterleavedARM: return "RowBlockedInterleavedARM"; + case CooperativeMatrixLayoutColumnBlockedInterleavedARM: return "ColumnBlockedInterleavedARM"; + default: return "Unknown"; + } +} + +inline const char* CooperativeMatrixUseToString(CooperativeMatrixUse value) { + switch (value) { + case CooperativeMatrixUseMatrixAKHR: return "MatrixAKHR"; + case CooperativeMatrixUseMatrixBKHR: return "MatrixBKHR"; + case CooperativeMatrixUseMatrixAccumulatorKHR: return "MatrixAccumulatorKHR"; + default: return "Unknown"; + } +} + +inline const char* TensorClampModeToString(TensorClampMode value) { + switch (value) { + case TensorClampModeUndefined: return "Undefined"; + case TensorClampModeConstant: return "Constant"; + case TensorClampModeClampToEdge: return "ClampToEdge"; + case TensorClampModeRepeat: return "Repeat"; + case TensorClampModeRepeatMirrored: return "RepeatMirrored"; + default: return "Unknown"; + } +} + +inline const char* InitializationModeQualifierToString(InitializationModeQualifier value) { + switch (value) { + case InitializationModeQualifierInitOnDeviceReprogramALTERA: return "InitOnDeviceReprogramALTERA"; + case InitializationModeQualifierInitOnDeviceResetALTERA: return "InitOnDeviceResetALTERA"; + default: return "Unknown"; + } +} + +inline const char* HostAccessQualifierToString(HostAccessQualifier value) { + switch (value) { + case HostAccessQualifierNoneINTEL: return "NoneINTEL"; + case HostAccessQualifierReadINTEL: return "ReadINTEL"; + case HostAccessQualifierWriteINTEL: return "WriteINTEL"; + case HostAccessQualifierReadWriteINTEL: return "ReadWriteINTEL"; + default: return "Unknown"; + } +} + +inline const char* LoadCacheControlToString(LoadCacheControl value) { + switch (value) { + case LoadCacheControlUncachedINTEL: return "UncachedINTEL"; + case LoadCacheControlCachedINTEL: return "CachedINTEL"; + case LoadCacheControlStreamingINTEL: return "StreamingINTEL"; + case LoadCacheControlInvalidateAfterReadINTEL: return "InvalidateAfterReadINTEL"; + case LoadCacheControlConstCachedINTEL: return "ConstCachedINTEL"; + default: return "Unknown"; + } +} + +inline const char* StoreCacheControlToString(StoreCacheControl value) { + switch (value) { + case StoreCacheControlUncachedINTEL: return "UncachedINTEL"; + case StoreCacheControlWriteThroughINTEL: return "WriteThroughINTEL"; + case StoreCacheControlWriteBackINTEL: return "WriteBackINTEL"; + case StoreCacheControlStreamingINTEL: return "StreamingINTEL"; + default: return "Unknown"; + } +} + +inline const char* NamedMaximumNumberOfRegistersToString(NamedMaximumNumberOfRegisters value) { + switch (value) { + case NamedMaximumNumberOfRegistersAutoINTEL: return "AutoINTEL"; + default: return "Unknown"; + } +} + +inline const char* FPEncodingToString(FPEncoding value) { + switch (value) { + case FPEncodingBFloat16KHR: return "BFloat16KHR"; + case FPEncodingFloat8E4M3EXT: return "Float8E4M3EXT"; + case FPEncodingFloat8E5M2EXT: return "Float8E5M2EXT"; + default: return "Unknown"; + } +} + +inline const char* CooperativeVectorMatrixLayoutToString(CooperativeVectorMatrixLayout value) { + switch (value) { + case CooperativeVectorMatrixLayoutRowMajorNV: return "RowMajorNV"; + case CooperativeVectorMatrixLayoutColumnMajorNV: return "ColumnMajorNV"; + case CooperativeVectorMatrixLayoutInferencingOptimalNV: return "InferencingOptimalNV"; + case CooperativeVectorMatrixLayoutTrainingOptimalNV: return "TrainingOptimalNV"; + default: return "Unknown"; + } +} + +inline const char* ComponentTypeToString(ComponentType value) { + switch (value) { + case ComponentTypeFloat16NV: return "Float16NV"; + case ComponentTypeFloat32NV: return "Float32NV"; + case ComponentTypeFloat64NV: return "Float64NV"; + case ComponentTypeSignedInt8NV: return "SignedInt8NV"; + case ComponentTypeSignedInt16NV: return "SignedInt16NV"; + case ComponentTypeSignedInt32NV: return "SignedInt32NV"; + case ComponentTypeSignedInt64NV: return "SignedInt64NV"; + case ComponentTypeUnsignedInt8NV: return "UnsignedInt8NV"; + case ComponentTypeUnsignedInt16NV: return "UnsignedInt16NV"; + case ComponentTypeUnsignedInt32NV: return "UnsignedInt32NV"; + case ComponentTypeUnsignedInt64NV: return "UnsignedInt64NV"; + case ComponentTypeSignedInt8PackedNV: return "SignedInt8PackedNV"; + case ComponentTypeUnsignedInt8PackedNV: return "UnsignedInt8PackedNV"; + case ComponentTypeFloatE4M3NV: return "FloatE4M3NV"; + case ComponentTypeFloatE5M2NV: return "FloatE5M2NV"; + default: return "Unknown"; + } +} + +inline const char* OpToString(Op value) { + switch (value) { + case OpNop: return "OpNop"; + case OpUndef: return "OpUndef"; + case OpSourceContinued: return "OpSourceContinued"; + case OpSource: return "OpSource"; + case OpSourceExtension: return "OpSourceExtension"; + case OpName: return "OpName"; + case OpMemberName: return "OpMemberName"; + case OpString: return "OpString"; + case OpLine: return "OpLine"; + case OpExtension: return "OpExtension"; + case OpExtInstImport: return "OpExtInstImport"; + case OpExtInst: return "OpExtInst"; + case OpMemoryModel: return "OpMemoryModel"; + case OpEntryPoint: return "OpEntryPoint"; + case OpExecutionMode: return "OpExecutionMode"; + case OpCapability: return "OpCapability"; + case OpTypeVoid: return "OpTypeVoid"; + case OpTypeBool: return "OpTypeBool"; + case OpTypeInt: return "OpTypeInt"; + case OpTypeFloat: return "OpTypeFloat"; + case OpTypeVector: return "OpTypeVector"; + case OpTypeMatrix: return "OpTypeMatrix"; + case OpTypeImage: return "OpTypeImage"; + case OpTypeSampler: return "OpTypeSampler"; + case OpTypeSampledImage: return "OpTypeSampledImage"; + case OpTypeArray: return "OpTypeArray"; + case OpTypeRuntimeArray: return "OpTypeRuntimeArray"; + case OpTypeStruct: return "OpTypeStruct"; + case OpTypeOpaque: return "OpTypeOpaque"; + case OpTypePointer: return "OpTypePointer"; + case OpTypeFunction: return "OpTypeFunction"; + case OpTypeEvent: return "OpTypeEvent"; + case OpTypeDeviceEvent: return "OpTypeDeviceEvent"; + case OpTypeReserveId: return "OpTypeReserveId"; + case OpTypeQueue: return "OpTypeQueue"; + case OpTypePipe: return "OpTypePipe"; + case OpTypeForwardPointer: return "OpTypeForwardPointer"; + case OpConstantTrue: return "OpConstantTrue"; + case OpConstantFalse: return "OpConstantFalse"; + case OpConstant: return "OpConstant"; + case OpConstantComposite: return "OpConstantComposite"; + case OpConstantSampler: return "OpConstantSampler"; + case OpConstantNull: return "OpConstantNull"; + case OpSpecConstantTrue: return "OpSpecConstantTrue"; + case OpSpecConstantFalse: return "OpSpecConstantFalse"; + case OpSpecConstant: return "OpSpecConstant"; + case OpSpecConstantComposite: return "OpSpecConstantComposite"; + case OpSpecConstantOp: return "OpSpecConstantOp"; + case OpFunction: return "OpFunction"; + case OpFunctionParameter: return "OpFunctionParameter"; + case OpFunctionEnd: return "OpFunctionEnd"; + case OpFunctionCall: return "OpFunctionCall"; + case OpVariable: return "OpVariable"; + case OpImageTexelPointer: return "OpImageTexelPointer"; + case OpLoad: return "OpLoad"; + case OpStore: return "OpStore"; + case OpCopyMemory: return "OpCopyMemory"; + case OpCopyMemorySized: return "OpCopyMemorySized"; + case OpAccessChain: return "OpAccessChain"; + case OpInBoundsAccessChain: return "OpInBoundsAccessChain"; + case OpPtrAccessChain: return "OpPtrAccessChain"; + case OpArrayLength: return "OpArrayLength"; + case OpGenericPtrMemSemantics: return "OpGenericPtrMemSemantics"; + case OpInBoundsPtrAccessChain: return "OpInBoundsPtrAccessChain"; + case OpDecorate: return "OpDecorate"; + case OpMemberDecorate: return "OpMemberDecorate"; + case OpDecorationGroup: return "OpDecorationGroup"; + case OpGroupDecorate: return "OpGroupDecorate"; + case OpGroupMemberDecorate: return "OpGroupMemberDecorate"; + case OpVectorExtractDynamic: return "OpVectorExtractDynamic"; + case OpVectorInsertDynamic: return "OpVectorInsertDynamic"; + case OpVectorShuffle: return "OpVectorShuffle"; + case OpCompositeConstruct: return "OpCompositeConstruct"; + case OpCompositeExtract: return "OpCompositeExtract"; + case OpCompositeInsert: return "OpCompositeInsert"; + case OpCopyObject: return "OpCopyObject"; + case OpTranspose: return "OpTranspose"; + case OpSampledImage: return "OpSampledImage"; + case OpImageSampleImplicitLod: return "OpImageSampleImplicitLod"; + case OpImageSampleExplicitLod: return "OpImageSampleExplicitLod"; + case OpImageSampleDrefImplicitLod: return "OpImageSampleDrefImplicitLod"; + case OpImageSampleDrefExplicitLod: return "OpImageSampleDrefExplicitLod"; + case OpImageSampleProjImplicitLod: return "OpImageSampleProjImplicitLod"; + case OpImageSampleProjExplicitLod: return "OpImageSampleProjExplicitLod"; + case OpImageSampleProjDrefImplicitLod: return "OpImageSampleProjDrefImplicitLod"; + case OpImageSampleProjDrefExplicitLod: return "OpImageSampleProjDrefExplicitLod"; + case OpImageFetch: return "OpImageFetch"; + case OpImageGather: return "OpImageGather"; + case OpImageDrefGather: return "OpImageDrefGather"; + case OpImageRead: return "OpImageRead"; + case OpImageWrite: return "OpImageWrite"; + case OpImage: return "OpImage"; + case OpImageQueryFormat: return "OpImageQueryFormat"; + case OpImageQueryOrder: return "OpImageQueryOrder"; + case OpImageQuerySizeLod: return "OpImageQuerySizeLod"; + case OpImageQuerySize: return "OpImageQuerySize"; + case OpImageQueryLod: return "OpImageQueryLod"; + case OpImageQueryLevels: return "OpImageQueryLevels"; + case OpImageQuerySamples: return "OpImageQuerySamples"; + case OpConvertFToU: return "OpConvertFToU"; + case OpConvertFToS: return "OpConvertFToS"; + case OpConvertSToF: return "OpConvertSToF"; + case OpConvertUToF: return "OpConvertUToF"; + case OpUConvert: return "OpUConvert"; + case OpSConvert: return "OpSConvert"; + case OpFConvert: return "OpFConvert"; + case OpQuantizeToF16: return "OpQuantizeToF16"; + case OpConvertPtrToU: return "OpConvertPtrToU"; + case OpSatConvertSToU: return "OpSatConvertSToU"; + case OpSatConvertUToS: return "OpSatConvertUToS"; + case OpConvertUToPtr: return "OpConvertUToPtr"; + case OpPtrCastToGeneric: return "OpPtrCastToGeneric"; + case OpGenericCastToPtr: return "OpGenericCastToPtr"; + case OpGenericCastToPtrExplicit: return "OpGenericCastToPtrExplicit"; + case OpBitcast: return "OpBitcast"; + case OpSNegate: return "OpSNegate"; + case OpFNegate: return "OpFNegate"; + case OpIAdd: return "OpIAdd"; + case OpFAdd: return "OpFAdd"; + case OpISub: return "OpISub"; + case OpFSub: return "OpFSub"; + case OpIMul: return "OpIMul"; + case OpFMul: return "OpFMul"; + case OpUDiv: return "OpUDiv"; + case OpSDiv: return "OpSDiv"; + case OpFDiv: return "OpFDiv"; + case OpUMod: return "OpUMod"; + case OpSRem: return "OpSRem"; + case OpSMod: return "OpSMod"; + case OpFRem: return "OpFRem"; + case OpFMod: return "OpFMod"; + case OpVectorTimesScalar: return "OpVectorTimesScalar"; + case OpMatrixTimesScalar: return "OpMatrixTimesScalar"; + case OpVectorTimesMatrix: return "OpVectorTimesMatrix"; + case OpMatrixTimesVector: return "OpMatrixTimesVector"; + case OpMatrixTimesMatrix: return "OpMatrixTimesMatrix"; + case OpOuterProduct: return "OpOuterProduct"; + case OpDot: return "OpDot"; + case OpIAddCarry: return "OpIAddCarry"; + case OpISubBorrow: return "OpISubBorrow"; + case OpUMulExtended: return "OpUMulExtended"; + case OpSMulExtended: return "OpSMulExtended"; + case OpAny: return "OpAny"; + case OpAll: return "OpAll"; + case OpIsNan: return "OpIsNan"; + case OpIsInf: return "OpIsInf"; + case OpIsFinite: return "OpIsFinite"; + case OpIsNormal: return "OpIsNormal"; + case OpSignBitSet: return "OpSignBitSet"; + case OpLessOrGreater: return "OpLessOrGreater"; + case OpOrdered: return "OpOrdered"; + case OpUnordered: return "OpUnordered"; + case OpLogicalEqual: return "OpLogicalEqual"; + case OpLogicalNotEqual: return "OpLogicalNotEqual"; + case OpLogicalOr: return "OpLogicalOr"; + case OpLogicalAnd: return "OpLogicalAnd"; + case OpLogicalNot: return "OpLogicalNot"; + case OpSelect: return "OpSelect"; + case OpIEqual: return "OpIEqual"; + case OpINotEqual: return "OpINotEqual"; + case OpUGreaterThan: return "OpUGreaterThan"; + case OpSGreaterThan: return "OpSGreaterThan"; + case OpUGreaterThanEqual: return "OpUGreaterThanEqual"; + case OpSGreaterThanEqual: return "OpSGreaterThanEqual"; + case OpULessThan: return "OpULessThan"; + case OpSLessThan: return "OpSLessThan"; + case OpULessThanEqual: return "OpULessThanEqual"; + case OpSLessThanEqual: return "OpSLessThanEqual"; + case OpFOrdEqual: return "OpFOrdEqual"; + case OpFUnordEqual: return "OpFUnordEqual"; + case OpFOrdNotEqual: return "OpFOrdNotEqual"; + case OpFUnordNotEqual: return "OpFUnordNotEqual"; + case OpFOrdLessThan: return "OpFOrdLessThan"; + case OpFUnordLessThan: return "OpFUnordLessThan"; + case OpFOrdGreaterThan: return "OpFOrdGreaterThan"; + case OpFUnordGreaterThan: return "OpFUnordGreaterThan"; + case OpFOrdLessThanEqual: return "OpFOrdLessThanEqual"; + case OpFUnordLessThanEqual: return "OpFUnordLessThanEqual"; + case OpFOrdGreaterThanEqual: return "OpFOrdGreaterThanEqual"; + case OpFUnordGreaterThanEqual: return "OpFUnordGreaterThanEqual"; + case OpShiftRightLogical: return "OpShiftRightLogical"; + case OpShiftRightArithmetic: return "OpShiftRightArithmetic"; + case OpShiftLeftLogical: return "OpShiftLeftLogical"; + case OpBitwiseOr: return "OpBitwiseOr"; + case OpBitwiseXor: return "OpBitwiseXor"; + case OpBitwiseAnd: return "OpBitwiseAnd"; + case OpNot: return "OpNot"; + case OpBitFieldInsert: return "OpBitFieldInsert"; + case OpBitFieldSExtract: return "OpBitFieldSExtract"; + case OpBitFieldUExtract: return "OpBitFieldUExtract"; + case OpBitReverse: return "OpBitReverse"; + case OpBitCount: return "OpBitCount"; + case OpDPdx: return "OpDPdx"; + case OpDPdy: return "OpDPdy"; + case OpFwidth: return "OpFwidth"; + case OpDPdxFine: return "OpDPdxFine"; + case OpDPdyFine: return "OpDPdyFine"; + case OpFwidthFine: return "OpFwidthFine"; + case OpDPdxCoarse: return "OpDPdxCoarse"; + case OpDPdyCoarse: return "OpDPdyCoarse"; + case OpFwidthCoarse: return "OpFwidthCoarse"; + case OpEmitVertex: return "OpEmitVertex"; + case OpEndPrimitive: return "OpEndPrimitive"; + case OpEmitStreamVertex: return "OpEmitStreamVertex"; + case OpEndStreamPrimitive: return "OpEndStreamPrimitive"; + case OpControlBarrier: return "OpControlBarrier"; + case OpMemoryBarrier: return "OpMemoryBarrier"; + case OpAtomicLoad: return "OpAtomicLoad"; + case OpAtomicStore: return "OpAtomicStore"; + case OpAtomicExchange: return "OpAtomicExchange"; + case OpAtomicCompareExchange: return "OpAtomicCompareExchange"; + case OpAtomicCompareExchangeWeak: return "OpAtomicCompareExchangeWeak"; + case OpAtomicIIncrement: return "OpAtomicIIncrement"; + case OpAtomicIDecrement: return "OpAtomicIDecrement"; + case OpAtomicIAdd: return "OpAtomicIAdd"; + case OpAtomicISub: return "OpAtomicISub"; + case OpAtomicSMin: return "OpAtomicSMin"; + case OpAtomicUMin: return "OpAtomicUMin"; + case OpAtomicSMax: return "OpAtomicSMax"; + case OpAtomicUMax: return "OpAtomicUMax"; + case OpAtomicAnd: return "OpAtomicAnd"; + case OpAtomicOr: return "OpAtomicOr"; + case OpAtomicXor: return "OpAtomicXor"; + case OpPhi: return "OpPhi"; + case OpLoopMerge: return "OpLoopMerge"; + case OpSelectionMerge: return "OpSelectionMerge"; + case OpLabel: return "OpLabel"; + case OpBranch: return "OpBranch"; + case OpBranchConditional: return "OpBranchConditional"; + case OpSwitch: return "OpSwitch"; + case OpKill: return "OpKill"; + case OpReturn: return "OpReturn"; + case OpReturnValue: return "OpReturnValue"; + case OpUnreachable: return "OpUnreachable"; + case OpLifetimeStart: return "OpLifetimeStart"; + case OpLifetimeStop: return "OpLifetimeStop"; + case OpGroupAsyncCopy: return "OpGroupAsyncCopy"; + case OpGroupWaitEvents: return "OpGroupWaitEvents"; + case OpGroupAll: return "OpGroupAll"; + case OpGroupAny: return "OpGroupAny"; + case OpGroupBroadcast: return "OpGroupBroadcast"; + case OpGroupIAdd: return "OpGroupIAdd"; + case OpGroupFAdd: return "OpGroupFAdd"; + case OpGroupFMin: return "OpGroupFMin"; + case OpGroupUMin: return "OpGroupUMin"; + case OpGroupSMin: return "OpGroupSMin"; + case OpGroupFMax: return "OpGroupFMax"; + case OpGroupUMax: return "OpGroupUMax"; + case OpGroupSMax: return "OpGroupSMax"; + case OpReadPipe: return "OpReadPipe"; + case OpWritePipe: return "OpWritePipe"; + case OpReservedReadPipe: return "OpReservedReadPipe"; + case OpReservedWritePipe: return "OpReservedWritePipe"; + case OpReserveReadPipePackets: return "OpReserveReadPipePackets"; + case OpReserveWritePipePackets: return "OpReserveWritePipePackets"; + case OpCommitReadPipe: return "OpCommitReadPipe"; + case OpCommitWritePipe: return "OpCommitWritePipe"; + case OpIsValidReserveId: return "OpIsValidReserveId"; + case OpGetNumPipePackets: return "OpGetNumPipePackets"; + case OpGetMaxPipePackets: return "OpGetMaxPipePackets"; + case OpGroupReserveReadPipePackets: return "OpGroupReserveReadPipePackets"; + case OpGroupReserveWritePipePackets: return "OpGroupReserveWritePipePackets"; + case OpGroupCommitReadPipe: return "OpGroupCommitReadPipe"; + case OpGroupCommitWritePipe: return "OpGroupCommitWritePipe"; + case OpEnqueueMarker: return "OpEnqueueMarker"; + case OpEnqueueKernel: return "OpEnqueueKernel"; + case OpGetKernelNDrangeSubGroupCount: return "OpGetKernelNDrangeSubGroupCount"; + case OpGetKernelNDrangeMaxSubGroupSize: return "OpGetKernelNDrangeMaxSubGroupSize"; + case OpGetKernelWorkGroupSize: return "OpGetKernelWorkGroupSize"; + case OpGetKernelPreferredWorkGroupSizeMultiple: return "OpGetKernelPreferredWorkGroupSizeMultiple"; + case OpRetainEvent: return "OpRetainEvent"; + case OpReleaseEvent: return "OpReleaseEvent"; + case OpCreateUserEvent: return "OpCreateUserEvent"; + case OpIsValidEvent: return "OpIsValidEvent"; + case OpSetUserEventStatus: return "OpSetUserEventStatus"; + case OpCaptureEventProfilingInfo: return "OpCaptureEventProfilingInfo"; + case OpGetDefaultQueue: return "OpGetDefaultQueue"; + case OpBuildNDRange: return "OpBuildNDRange"; + case OpImageSparseSampleImplicitLod: return "OpImageSparseSampleImplicitLod"; + case OpImageSparseSampleExplicitLod: return "OpImageSparseSampleExplicitLod"; + case OpImageSparseSampleDrefImplicitLod: return "OpImageSparseSampleDrefImplicitLod"; + case OpImageSparseSampleDrefExplicitLod: return "OpImageSparseSampleDrefExplicitLod"; + case OpImageSparseSampleProjImplicitLod: return "OpImageSparseSampleProjImplicitLod"; + case OpImageSparseSampleProjExplicitLod: return "OpImageSparseSampleProjExplicitLod"; + case OpImageSparseSampleProjDrefImplicitLod: return "OpImageSparseSampleProjDrefImplicitLod"; + case OpImageSparseSampleProjDrefExplicitLod: return "OpImageSparseSampleProjDrefExplicitLod"; + case OpImageSparseFetch: return "OpImageSparseFetch"; + case OpImageSparseGather: return "OpImageSparseGather"; + case OpImageSparseDrefGather: return "OpImageSparseDrefGather"; + case OpImageSparseTexelsResident: return "OpImageSparseTexelsResident"; + case OpNoLine: return "OpNoLine"; + case OpAtomicFlagTestAndSet: return "OpAtomicFlagTestAndSet"; + case OpAtomicFlagClear: return "OpAtomicFlagClear"; + case OpImageSparseRead: return "OpImageSparseRead"; + case OpSizeOf: return "OpSizeOf"; + case OpTypePipeStorage: return "OpTypePipeStorage"; + case OpConstantPipeStorage: return "OpConstantPipeStorage"; + case OpCreatePipeFromPipeStorage: return "OpCreatePipeFromPipeStorage"; + case OpGetKernelLocalSizeForSubgroupCount: return "OpGetKernelLocalSizeForSubgroupCount"; + case OpGetKernelMaxNumSubgroups: return "OpGetKernelMaxNumSubgroups"; + case OpTypeNamedBarrier: return "OpTypeNamedBarrier"; + case OpNamedBarrierInitialize: return "OpNamedBarrierInitialize"; + case OpMemoryNamedBarrier: return "OpMemoryNamedBarrier"; + case OpModuleProcessed: return "OpModuleProcessed"; + case OpExecutionModeId: return "OpExecutionModeId"; + case OpDecorateId: return "OpDecorateId"; + case OpGroupNonUniformElect: return "OpGroupNonUniformElect"; + case OpGroupNonUniformAll: return "OpGroupNonUniformAll"; + case OpGroupNonUniformAny: return "OpGroupNonUniformAny"; + case OpGroupNonUniformAllEqual: return "OpGroupNonUniformAllEqual"; + case OpGroupNonUniformBroadcast: return "OpGroupNonUniformBroadcast"; + case OpGroupNonUniformBroadcastFirst: return "OpGroupNonUniformBroadcastFirst"; + case OpGroupNonUniformBallot: return "OpGroupNonUniformBallot"; + case OpGroupNonUniformInverseBallot: return "OpGroupNonUniformInverseBallot"; + case OpGroupNonUniformBallotBitExtract: return "OpGroupNonUniformBallotBitExtract"; + case OpGroupNonUniformBallotBitCount: return "OpGroupNonUniformBallotBitCount"; + case OpGroupNonUniformBallotFindLSB: return "OpGroupNonUniformBallotFindLSB"; + case OpGroupNonUniformBallotFindMSB: return "OpGroupNonUniformBallotFindMSB"; + case OpGroupNonUniformShuffle: return "OpGroupNonUniformShuffle"; + case OpGroupNonUniformShuffleXor: return "OpGroupNonUniformShuffleXor"; + case OpGroupNonUniformShuffleUp: return "OpGroupNonUniformShuffleUp"; + case OpGroupNonUniformShuffleDown: return "OpGroupNonUniformShuffleDown"; + case OpGroupNonUniformIAdd: return "OpGroupNonUniformIAdd"; + case OpGroupNonUniformFAdd: return "OpGroupNonUniformFAdd"; + case OpGroupNonUniformIMul: return "OpGroupNonUniformIMul"; + case OpGroupNonUniformFMul: return "OpGroupNonUniformFMul"; + case OpGroupNonUniformSMin: return "OpGroupNonUniformSMin"; + case OpGroupNonUniformUMin: return "OpGroupNonUniformUMin"; + case OpGroupNonUniformFMin: return "OpGroupNonUniformFMin"; + case OpGroupNonUniformSMax: return "OpGroupNonUniformSMax"; + case OpGroupNonUniformUMax: return "OpGroupNonUniformUMax"; + case OpGroupNonUniformFMax: return "OpGroupNonUniformFMax"; + case OpGroupNonUniformBitwiseAnd: return "OpGroupNonUniformBitwiseAnd"; + case OpGroupNonUniformBitwiseOr: return "OpGroupNonUniformBitwiseOr"; + case OpGroupNonUniformBitwiseXor: return "OpGroupNonUniformBitwiseXor"; + case OpGroupNonUniformLogicalAnd: return "OpGroupNonUniformLogicalAnd"; + case OpGroupNonUniformLogicalOr: return "OpGroupNonUniformLogicalOr"; + case OpGroupNonUniformLogicalXor: return "OpGroupNonUniformLogicalXor"; + case OpGroupNonUniformQuadBroadcast: return "OpGroupNonUniformQuadBroadcast"; + case OpGroupNonUniformQuadSwap: return "OpGroupNonUniformQuadSwap"; + case OpCopyLogical: return "OpCopyLogical"; + case OpPtrEqual: return "OpPtrEqual"; + case OpPtrNotEqual: return "OpPtrNotEqual"; + case OpPtrDiff: return "OpPtrDiff"; + case OpColorAttachmentReadEXT: return "OpColorAttachmentReadEXT"; + case OpDepthAttachmentReadEXT: return "OpDepthAttachmentReadEXT"; + case OpStencilAttachmentReadEXT: return "OpStencilAttachmentReadEXT"; + case OpTypeTensorARM: return "OpTypeTensorARM"; + case OpTensorReadARM: return "OpTensorReadARM"; + case OpTensorWriteARM: return "OpTensorWriteARM"; + case OpTensorQuerySizeARM: return "OpTensorQuerySizeARM"; + case OpGraphConstantARM: return "OpGraphConstantARM"; + case OpGraphEntryPointARM: return "OpGraphEntryPointARM"; + case OpGraphARM: return "OpGraphARM"; + case OpGraphInputARM: return "OpGraphInputARM"; + case OpGraphSetOutputARM: return "OpGraphSetOutputARM"; + case OpGraphEndARM: return "OpGraphEndARM"; + case OpTypeGraphARM: return "OpTypeGraphARM"; + case OpTerminateInvocation: return "OpTerminateInvocation"; + case OpTypeUntypedPointerKHR: return "OpTypeUntypedPointerKHR"; + case OpUntypedVariableKHR: return "OpUntypedVariableKHR"; + case OpUntypedAccessChainKHR: return "OpUntypedAccessChainKHR"; + case OpUntypedInBoundsAccessChainKHR: return "OpUntypedInBoundsAccessChainKHR"; + case OpSubgroupBallotKHR: return "OpSubgroupBallotKHR"; + case OpSubgroupFirstInvocationKHR: return "OpSubgroupFirstInvocationKHR"; + case OpUntypedPtrAccessChainKHR: return "OpUntypedPtrAccessChainKHR"; + case OpUntypedInBoundsPtrAccessChainKHR: return "OpUntypedInBoundsPtrAccessChainKHR"; + case OpUntypedArrayLengthKHR: return "OpUntypedArrayLengthKHR"; + case OpUntypedPrefetchKHR: return "OpUntypedPrefetchKHR"; + case OpFmaKHR: return "OpFmaKHR"; + case OpSubgroupAllKHR: return "OpSubgroupAllKHR"; + case OpSubgroupAnyKHR: return "OpSubgroupAnyKHR"; + case OpSubgroupAllEqualKHR: return "OpSubgroupAllEqualKHR"; + case OpGroupNonUniformRotateKHR: return "OpGroupNonUniformRotateKHR"; + case OpSubgroupReadInvocationKHR: return "OpSubgroupReadInvocationKHR"; + case OpExtInstWithForwardRefsKHR: return "OpExtInstWithForwardRefsKHR"; + case OpUntypedGroupAsyncCopyKHR: return "OpUntypedGroupAsyncCopyKHR"; + case OpTraceRayKHR: return "OpTraceRayKHR"; + case OpExecuteCallableKHR: return "OpExecuteCallableKHR"; + case OpConvertUToAccelerationStructureKHR: return "OpConvertUToAccelerationStructureKHR"; + case OpIgnoreIntersectionKHR: return "OpIgnoreIntersectionKHR"; + case OpTerminateRayKHR: return "OpTerminateRayKHR"; + case OpSDot: return "OpSDot"; + case OpUDot: return "OpUDot"; + case OpSUDot: return "OpSUDot"; + case OpSDotAccSat: return "OpSDotAccSat"; + case OpUDotAccSat: return "OpUDotAccSat"; + case OpSUDotAccSat: return "OpSUDotAccSat"; + case OpTypeCooperativeMatrixKHR: return "OpTypeCooperativeMatrixKHR"; + case OpCooperativeMatrixLoadKHR: return "OpCooperativeMatrixLoadKHR"; + case OpCooperativeMatrixStoreKHR: return "OpCooperativeMatrixStoreKHR"; + case OpCooperativeMatrixMulAddKHR: return "OpCooperativeMatrixMulAddKHR"; + case OpCooperativeMatrixLengthKHR: return "OpCooperativeMatrixLengthKHR"; + case OpConstantCompositeReplicateEXT: return "OpConstantCompositeReplicateEXT"; + case OpSpecConstantCompositeReplicateEXT: return "OpSpecConstantCompositeReplicateEXT"; + case OpCompositeConstructReplicateEXT: return "OpCompositeConstructReplicateEXT"; + case OpTypeRayQueryKHR: return "OpTypeRayQueryKHR"; + case OpRayQueryInitializeKHR: return "OpRayQueryInitializeKHR"; + case OpRayQueryTerminateKHR: return "OpRayQueryTerminateKHR"; + case OpRayQueryGenerateIntersectionKHR: return "OpRayQueryGenerateIntersectionKHR"; + case OpRayQueryConfirmIntersectionKHR: return "OpRayQueryConfirmIntersectionKHR"; + case OpRayQueryProceedKHR: return "OpRayQueryProceedKHR"; + case OpRayQueryGetIntersectionTypeKHR: return "OpRayQueryGetIntersectionTypeKHR"; + case OpImageSampleWeightedQCOM: return "OpImageSampleWeightedQCOM"; + case OpImageBoxFilterQCOM: return "OpImageBoxFilterQCOM"; + case OpImageBlockMatchSSDQCOM: return "OpImageBlockMatchSSDQCOM"; + case OpImageBlockMatchSADQCOM: return "OpImageBlockMatchSADQCOM"; + case OpBitCastArrayQCOM: return "OpBitCastArrayQCOM"; + case OpImageBlockMatchWindowSSDQCOM: return "OpImageBlockMatchWindowSSDQCOM"; + case OpImageBlockMatchWindowSADQCOM: return "OpImageBlockMatchWindowSADQCOM"; + case OpImageBlockMatchGatherSSDQCOM: return "OpImageBlockMatchGatherSSDQCOM"; + case OpImageBlockMatchGatherSADQCOM: return "OpImageBlockMatchGatherSADQCOM"; + case OpCompositeConstructCoopMatQCOM: return "OpCompositeConstructCoopMatQCOM"; + case OpCompositeExtractCoopMatQCOM: return "OpCompositeExtractCoopMatQCOM"; + case OpExtractSubArrayQCOM: return "OpExtractSubArrayQCOM"; + case OpGroupIAddNonUniformAMD: return "OpGroupIAddNonUniformAMD"; + case OpGroupFAddNonUniformAMD: return "OpGroupFAddNonUniformAMD"; + case OpGroupFMinNonUniformAMD: return "OpGroupFMinNonUniformAMD"; + case OpGroupUMinNonUniformAMD: return "OpGroupUMinNonUniformAMD"; + case OpGroupSMinNonUniformAMD: return "OpGroupSMinNonUniformAMD"; + case OpGroupFMaxNonUniformAMD: return "OpGroupFMaxNonUniformAMD"; + case OpGroupUMaxNonUniformAMD: return "OpGroupUMaxNonUniformAMD"; + case OpGroupSMaxNonUniformAMD: return "OpGroupSMaxNonUniformAMD"; + case OpFragmentMaskFetchAMD: return "OpFragmentMaskFetchAMD"; + case OpFragmentFetchAMD: return "OpFragmentFetchAMD"; + case OpReadClockKHR: return "OpReadClockKHR"; + case OpAllocateNodePayloadsAMDX: return "OpAllocateNodePayloadsAMDX"; + case OpEnqueueNodePayloadsAMDX: return "OpEnqueueNodePayloadsAMDX"; + case OpTypeNodePayloadArrayAMDX: return "OpTypeNodePayloadArrayAMDX"; + case OpFinishWritingNodePayloadAMDX: return "OpFinishWritingNodePayloadAMDX"; + case OpNodePayloadArrayLengthAMDX: return "OpNodePayloadArrayLengthAMDX"; + case OpIsNodePayloadValidAMDX: return "OpIsNodePayloadValidAMDX"; + case OpConstantStringAMDX: return "OpConstantStringAMDX"; + case OpSpecConstantStringAMDX: return "OpSpecConstantStringAMDX"; + case OpGroupNonUniformQuadAllKHR: return "OpGroupNonUniformQuadAllKHR"; + case OpGroupNonUniformQuadAnyKHR: return "OpGroupNonUniformQuadAnyKHR"; + case OpTypeBufferEXT: return "OpTypeBufferEXT"; + case OpBufferPointerEXT: return "OpBufferPointerEXT"; + case OpAbortKHR: return "OpAbortKHR"; + case OpUntypedImageTexelPointerEXT: return "OpUntypedImageTexelPointerEXT"; + case OpMemberDecorateIdEXT: return "OpMemberDecorateIdEXT"; + case OpConstantSizeOfEXT: return "OpConstantSizeOfEXT"; + case OpConstantDataKHR: return "OpConstantDataKHR"; + case OpSpecConstantDataKHR: return "OpSpecConstantDataKHR"; + case OpPoisonKHR: return "OpPoisonKHR"; + case OpFreezeKHR: return "OpFreezeKHR"; + case OpHitObjectRecordHitMotionNV: return "OpHitObjectRecordHitMotionNV"; + case OpHitObjectRecordHitWithIndexMotionNV: return "OpHitObjectRecordHitWithIndexMotionNV"; + case OpHitObjectRecordMissMotionNV: return "OpHitObjectRecordMissMotionNV"; + case OpHitObjectGetWorldToObjectNV: return "OpHitObjectGetWorldToObjectNV"; + case OpHitObjectGetObjectToWorldNV: return "OpHitObjectGetObjectToWorldNV"; + case OpHitObjectGetObjectRayDirectionNV: return "OpHitObjectGetObjectRayDirectionNV"; + case OpHitObjectGetObjectRayOriginNV: return "OpHitObjectGetObjectRayOriginNV"; + case OpHitObjectTraceRayMotionNV: return "OpHitObjectTraceRayMotionNV"; + case OpHitObjectGetShaderRecordBufferHandleNV: return "OpHitObjectGetShaderRecordBufferHandleNV"; + case OpHitObjectGetShaderBindingTableRecordIndexNV: return "OpHitObjectGetShaderBindingTableRecordIndexNV"; + case OpHitObjectRecordEmptyNV: return "OpHitObjectRecordEmptyNV"; + case OpHitObjectTraceRayNV: return "OpHitObjectTraceRayNV"; + case OpHitObjectRecordHitNV: return "OpHitObjectRecordHitNV"; + case OpHitObjectRecordHitWithIndexNV: return "OpHitObjectRecordHitWithIndexNV"; + case OpHitObjectRecordMissNV: return "OpHitObjectRecordMissNV"; + case OpHitObjectExecuteShaderNV: return "OpHitObjectExecuteShaderNV"; + case OpHitObjectGetCurrentTimeNV: return "OpHitObjectGetCurrentTimeNV"; + case OpHitObjectGetAttributesNV: return "OpHitObjectGetAttributesNV"; + case OpHitObjectGetHitKindNV: return "OpHitObjectGetHitKindNV"; + case OpHitObjectGetPrimitiveIndexNV: return "OpHitObjectGetPrimitiveIndexNV"; + case OpHitObjectGetGeometryIndexNV: return "OpHitObjectGetGeometryIndexNV"; + case OpHitObjectGetInstanceIdNV: return "OpHitObjectGetInstanceIdNV"; + case OpHitObjectGetInstanceCustomIndexNV: return "OpHitObjectGetInstanceCustomIndexNV"; + case OpHitObjectGetWorldRayDirectionNV: return "OpHitObjectGetWorldRayDirectionNV"; + case OpHitObjectGetWorldRayOriginNV: return "OpHitObjectGetWorldRayOriginNV"; + case OpHitObjectGetRayTMaxNV: return "OpHitObjectGetRayTMaxNV"; + case OpHitObjectGetRayTMinNV: return "OpHitObjectGetRayTMinNV"; + case OpHitObjectIsEmptyNV: return "OpHitObjectIsEmptyNV"; + case OpHitObjectIsHitNV: return "OpHitObjectIsHitNV"; + case OpHitObjectIsMissNV: return "OpHitObjectIsMissNV"; + case OpReorderThreadWithHitObjectNV: return "OpReorderThreadWithHitObjectNV"; + case OpReorderThreadWithHintNV: return "OpReorderThreadWithHintNV"; + case OpTypeHitObjectNV: return "OpTypeHitObjectNV"; + case OpImageSampleFootprintNV: return "OpImageSampleFootprintNV"; + case OpTypeCooperativeVectorNV: return "OpTypeCooperativeVectorNV"; + case OpCooperativeVectorMatrixMulNV: return "OpCooperativeVectorMatrixMulNV"; + case OpCooperativeVectorOuterProductAccumulateNV: return "OpCooperativeVectorOuterProductAccumulateNV"; + case OpCooperativeVectorReduceSumAccumulateNV: return "OpCooperativeVectorReduceSumAccumulateNV"; + case OpCooperativeVectorMatrixMulAddNV: return "OpCooperativeVectorMatrixMulAddNV"; + case OpCooperativeMatrixConvertNV: return "OpCooperativeMatrixConvertNV"; + case OpEmitMeshTasksEXT: return "OpEmitMeshTasksEXT"; + case OpSetMeshOutputsEXT: return "OpSetMeshOutputsEXT"; + case OpGroupNonUniformPartitionEXT: return "OpGroupNonUniformPartitionEXT"; + case OpWritePackedPrimitiveIndices4x8NV: return "OpWritePackedPrimitiveIndices4x8NV"; + case OpFetchMicroTriangleVertexPositionNV: return "OpFetchMicroTriangleVertexPositionNV"; + case OpFetchMicroTriangleVertexBarycentricNV: return "OpFetchMicroTriangleVertexBarycentricNV"; + case OpCooperativeVectorLoadNV: return "OpCooperativeVectorLoadNV"; + case OpCooperativeVectorStoreNV: return "OpCooperativeVectorStoreNV"; + case OpHitObjectRecordFromQueryEXT: return "OpHitObjectRecordFromQueryEXT"; + case OpHitObjectRecordMissEXT: return "OpHitObjectRecordMissEXT"; + case OpHitObjectRecordMissMotionEXT: return "OpHitObjectRecordMissMotionEXT"; + case OpHitObjectGetIntersectionTriangleVertexPositionsEXT: return "OpHitObjectGetIntersectionTriangleVertexPositionsEXT"; + case OpHitObjectGetRayFlagsEXT: return "OpHitObjectGetRayFlagsEXT"; + case OpHitObjectSetShaderBindingTableRecordIndexEXT: return "OpHitObjectSetShaderBindingTableRecordIndexEXT"; + case OpHitObjectReorderExecuteShaderEXT: return "OpHitObjectReorderExecuteShaderEXT"; + case OpHitObjectTraceReorderExecuteEXT: return "OpHitObjectTraceReorderExecuteEXT"; + case OpHitObjectTraceMotionReorderExecuteEXT: return "OpHitObjectTraceMotionReorderExecuteEXT"; + case OpTypeHitObjectEXT: return "OpTypeHitObjectEXT"; + case OpReorderThreadWithHintEXT: return "OpReorderThreadWithHintEXT"; + case OpReorderThreadWithHitObjectEXT: return "OpReorderThreadWithHitObjectEXT"; + case OpHitObjectTraceRayEXT: return "OpHitObjectTraceRayEXT"; + case OpHitObjectTraceRayMotionEXT: return "OpHitObjectTraceRayMotionEXT"; + case OpHitObjectRecordEmptyEXT: return "OpHitObjectRecordEmptyEXT"; + case OpHitObjectExecuteShaderEXT: return "OpHitObjectExecuteShaderEXT"; + case OpHitObjectGetCurrentTimeEXT: return "OpHitObjectGetCurrentTimeEXT"; + case OpHitObjectGetAttributesEXT: return "OpHitObjectGetAttributesEXT"; + case OpHitObjectGetHitKindEXT: return "OpHitObjectGetHitKindEXT"; + case OpHitObjectGetPrimitiveIndexEXT: return "OpHitObjectGetPrimitiveIndexEXT"; + case OpHitObjectGetGeometryIndexEXT: return "OpHitObjectGetGeometryIndexEXT"; + case OpHitObjectGetInstanceIdEXT: return "OpHitObjectGetInstanceIdEXT"; + case OpHitObjectGetInstanceCustomIndexEXT: return "OpHitObjectGetInstanceCustomIndexEXT"; + case OpHitObjectGetObjectRayOriginEXT: return "OpHitObjectGetObjectRayOriginEXT"; + case OpHitObjectGetObjectRayDirectionEXT: return "OpHitObjectGetObjectRayDirectionEXT"; + case OpHitObjectGetWorldRayDirectionEXT: return "OpHitObjectGetWorldRayDirectionEXT"; + case OpHitObjectGetWorldRayOriginEXT: return "OpHitObjectGetWorldRayOriginEXT"; + case OpHitObjectGetObjectToWorldEXT: return "OpHitObjectGetObjectToWorldEXT"; + case OpHitObjectGetWorldToObjectEXT: return "OpHitObjectGetWorldToObjectEXT"; + case OpHitObjectGetRayTMaxEXT: return "OpHitObjectGetRayTMaxEXT"; + case OpReportIntersectionKHR: return "OpReportIntersectionKHR"; + case OpIgnoreIntersectionNV: return "OpIgnoreIntersectionNV"; + case OpTerminateRayNV: return "OpTerminateRayNV"; + case OpTraceNV: return "OpTraceNV"; + case OpTraceMotionNV: return "OpTraceMotionNV"; + case OpTraceRayMotionNV: return "OpTraceRayMotionNV"; + case OpRayQueryGetIntersectionTriangleVertexPositionsKHR: return "OpRayQueryGetIntersectionTriangleVertexPositionsKHR"; + case OpTypeAccelerationStructureKHR: return "OpTypeAccelerationStructureKHR"; + case OpExecuteCallableNV: return "OpExecuteCallableNV"; + case OpRayQueryGetClusterIdNV: return "OpRayQueryGetClusterIdNV"; + case OpHitObjectGetClusterIdNV: return "OpHitObjectGetClusterIdNV"; + case OpHitObjectGetRayTMinEXT: return "OpHitObjectGetRayTMinEXT"; + case OpHitObjectGetShaderBindingTableRecordIndexEXT: return "OpHitObjectGetShaderBindingTableRecordIndexEXT"; + case OpHitObjectGetShaderRecordBufferHandleEXT: return "OpHitObjectGetShaderRecordBufferHandleEXT"; + case OpHitObjectIsEmptyEXT: return "OpHitObjectIsEmptyEXT"; + case OpHitObjectIsHitEXT: return "OpHitObjectIsHitEXT"; + case OpHitObjectIsMissEXT: return "OpHitObjectIsMissEXT"; + case OpTypeCooperativeMatrixNV: return "OpTypeCooperativeMatrixNV"; + case OpCooperativeMatrixLoadNV: return "OpCooperativeMatrixLoadNV"; + case OpCooperativeMatrixStoreNV: return "OpCooperativeMatrixStoreNV"; + case OpCooperativeMatrixMulAddNV: return "OpCooperativeMatrixMulAddNV"; + case OpCooperativeMatrixLengthNV: return "OpCooperativeMatrixLengthNV"; + case OpBeginInvocationInterlockEXT: return "OpBeginInvocationInterlockEXT"; + case OpEndInvocationInterlockEXT: return "OpEndInvocationInterlockEXT"; + case OpCooperativeMatrixReduceNV: return "OpCooperativeMatrixReduceNV"; + case OpCooperativeMatrixLoadTensorNV: return "OpCooperativeMatrixLoadTensorNV"; + case OpCooperativeMatrixStoreTensorNV: return "OpCooperativeMatrixStoreTensorNV"; + case OpCooperativeMatrixPerElementOpNV: return "OpCooperativeMatrixPerElementOpNV"; + case OpTypeTensorLayoutNV: return "OpTypeTensorLayoutNV"; + case OpTypeTensorViewNV: return "OpTypeTensorViewNV"; + case OpCreateTensorLayoutNV: return "OpCreateTensorLayoutNV"; + case OpTensorLayoutSetDimensionNV: return "OpTensorLayoutSetDimensionNV"; + case OpTensorLayoutSetStrideNV: return "OpTensorLayoutSetStrideNV"; + case OpTensorLayoutSliceNV: return "OpTensorLayoutSliceNV"; + case OpTensorLayoutSetClampValueNV: return "OpTensorLayoutSetClampValueNV"; + case OpCreateTensorViewNV: return "OpCreateTensorViewNV"; + case OpTensorViewSetDimensionNV: return "OpTensorViewSetDimensionNV"; + case OpTensorViewSetStrideNV: return "OpTensorViewSetStrideNV"; + case OpDemoteToHelperInvocation: return "OpDemoteToHelperInvocation"; + case OpIsHelperInvocationEXT: return "OpIsHelperInvocationEXT"; + case OpTensorViewSetClipNV: return "OpTensorViewSetClipNV"; + case OpTensorLayoutSetBlockSizeNV: return "OpTensorLayoutSetBlockSizeNV"; + case OpCooperativeMatrixTransposeNV: return "OpCooperativeMatrixTransposeNV"; + case OpConvertUToImageNV: return "OpConvertUToImageNV"; + case OpConvertUToSamplerNV: return "OpConvertUToSamplerNV"; + case OpConvertImageToUNV: return "OpConvertImageToUNV"; + case OpConvertSamplerToUNV: return "OpConvertSamplerToUNV"; + case OpConvertUToSampledImageNV: return "OpConvertUToSampledImageNV"; + case OpConvertSampledImageToUNV: return "OpConvertSampledImageToUNV"; + case OpSamplerImageAddressingModeNV: return "OpSamplerImageAddressingModeNV"; + case OpRawAccessChainNV: return "OpRawAccessChainNV"; + case OpRayQueryGetIntersectionSpherePositionNV: return "OpRayQueryGetIntersectionSpherePositionNV"; + case OpRayQueryGetIntersectionSphereRadiusNV: return "OpRayQueryGetIntersectionSphereRadiusNV"; + case OpRayQueryGetIntersectionLSSPositionsNV: return "OpRayQueryGetIntersectionLSSPositionsNV"; + case OpRayQueryGetIntersectionLSSRadiiNV: return "OpRayQueryGetIntersectionLSSRadiiNV"; + case OpRayQueryGetIntersectionLSSHitValueNV: return "OpRayQueryGetIntersectionLSSHitValueNV"; + case OpHitObjectGetSpherePositionNV: return "OpHitObjectGetSpherePositionNV"; + case OpHitObjectGetSphereRadiusNV: return "OpHitObjectGetSphereRadiusNV"; + case OpHitObjectGetLSSPositionsNV: return "OpHitObjectGetLSSPositionsNV"; + case OpHitObjectGetLSSRadiiNV: return "OpHitObjectGetLSSRadiiNV"; + case OpHitObjectIsSphereHitNV: return "OpHitObjectIsSphereHitNV"; + case OpHitObjectIsLSSHitNV: return "OpHitObjectIsLSSHitNV"; + case OpRayQueryIsSphereHitNV: return "OpRayQueryIsSphereHitNV"; + case OpRayQueryIsLSSHitNV: return "OpRayQueryIsLSSHitNV"; + case OpSubgroupShuffleINTEL: return "OpSubgroupShuffleINTEL"; + case OpSubgroupShuffleDownINTEL: return "OpSubgroupShuffleDownINTEL"; + case OpSubgroupShuffleUpINTEL: return "OpSubgroupShuffleUpINTEL"; + case OpSubgroupShuffleXorINTEL: return "OpSubgroupShuffleXorINTEL"; + case OpSubgroupBlockReadINTEL: return "OpSubgroupBlockReadINTEL"; + case OpSubgroupBlockWriteINTEL: return "OpSubgroupBlockWriteINTEL"; + case OpSubgroupImageBlockReadINTEL: return "OpSubgroupImageBlockReadINTEL"; + case OpSubgroupImageBlockWriteINTEL: return "OpSubgroupImageBlockWriteINTEL"; + case OpSubgroupImageMediaBlockReadINTEL: return "OpSubgroupImageMediaBlockReadINTEL"; + case OpSubgroupImageMediaBlockWriteINTEL: return "OpSubgroupImageMediaBlockWriteINTEL"; + case OpUCountLeadingZerosINTEL: return "OpUCountLeadingZerosINTEL"; + case OpUCountTrailingZerosINTEL: return "OpUCountTrailingZerosINTEL"; + case OpAbsISubINTEL: return "OpAbsISubINTEL"; + case OpAbsUSubINTEL: return "OpAbsUSubINTEL"; + case OpIAddSatINTEL: return "OpIAddSatINTEL"; + case OpUAddSatINTEL: return "OpUAddSatINTEL"; + case OpIAverageINTEL: return "OpIAverageINTEL"; + case OpUAverageINTEL: return "OpUAverageINTEL"; + case OpIAverageRoundedINTEL: return "OpIAverageRoundedINTEL"; + case OpUAverageRoundedINTEL: return "OpUAverageRoundedINTEL"; + case OpISubSatINTEL: return "OpISubSatINTEL"; + case OpUSubSatINTEL: return "OpUSubSatINTEL"; + case OpIMul32x16INTEL: return "OpIMul32x16INTEL"; + case OpUMul32x16INTEL: return "OpUMul32x16INTEL"; + case OpConstantFunctionPointerINTEL: return "OpConstantFunctionPointerINTEL"; + case OpFunctionPointerCallINTEL: return "OpFunctionPointerCallINTEL"; + case OpAsmTargetINTEL: return "OpAsmTargetINTEL"; + case OpAsmINTEL: return "OpAsmINTEL"; + case OpAsmCallINTEL: return "OpAsmCallINTEL"; + case OpAtomicFMinEXT: return "OpAtomicFMinEXT"; + case OpAtomicFMaxEXT: return "OpAtomicFMaxEXT"; + case OpAssumeTrueKHR: return "OpAssumeTrueKHR"; + case OpExpectKHR: return "OpExpectKHR"; + case OpDecorateString: return "OpDecorateString"; + case OpMemberDecorateString: return "OpMemberDecorateString"; + case OpVmeImageINTEL: return "OpVmeImageINTEL"; + case OpTypeVmeImageINTEL: return "OpTypeVmeImageINTEL"; + case OpTypeAvcImePayloadINTEL: return "OpTypeAvcImePayloadINTEL"; + case OpTypeAvcRefPayloadINTEL: return "OpTypeAvcRefPayloadINTEL"; + case OpTypeAvcSicPayloadINTEL: return "OpTypeAvcSicPayloadINTEL"; + case OpTypeAvcMcePayloadINTEL: return "OpTypeAvcMcePayloadINTEL"; + case OpTypeAvcMceResultINTEL: return "OpTypeAvcMceResultINTEL"; + case OpTypeAvcImeResultINTEL: return "OpTypeAvcImeResultINTEL"; + case OpTypeAvcImeResultSingleReferenceStreamoutINTEL: return "OpTypeAvcImeResultSingleReferenceStreamoutINTEL"; + case OpTypeAvcImeResultDualReferenceStreamoutINTEL: return "OpTypeAvcImeResultDualReferenceStreamoutINTEL"; + case OpTypeAvcImeSingleReferenceStreaminINTEL: return "OpTypeAvcImeSingleReferenceStreaminINTEL"; + case OpTypeAvcImeDualReferenceStreaminINTEL: return "OpTypeAvcImeDualReferenceStreaminINTEL"; + case OpTypeAvcRefResultINTEL: return "OpTypeAvcRefResultINTEL"; + case OpTypeAvcSicResultINTEL: return "OpTypeAvcSicResultINTEL"; + case OpSubgroupAvcMceGetDefaultInterBaseMultiReferencePenaltyINTEL: return "OpSubgroupAvcMceGetDefaultInterBaseMultiReferencePenaltyINTEL"; + case OpSubgroupAvcMceSetInterBaseMultiReferencePenaltyINTEL: return "OpSubgroupAvcMceSetInterBaseMultiReferencePenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultInterShapePenaltyINTEL: return "OpSubgroupAvcMceGetDefaultInterShapePenaltyINTEL"; + case OpSubgroupAvcMceSetInterShapePenaltyINTEL: return "OpSubgroupAvcMceSetInterShapePenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultInterDirectionPenaltyINTEL: return "OpSubgroupAvcMceGetDefaultInterDirectionPenaltyINTEL"; + case OpSubgroupAvcMceSetInterDirectionPenaltyINTEL: return "OpSubgroupAvcMceSetInterDirectionPenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultIntraLumaShapePenaltyINTEL: return "OpSubgroupAvcMceGetDefaultIntraLumaShapePenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultInterMotionVectorCostTableINTEL: return "OpSubgroupAvcMceGetDefaultInterMotionVectorCostTableINTEL"; + case OpSubgroupAvcMceGetDefaultHighPenaltyCostTableINTEL: return "OpSubgroupAvcMceGetDefaultHighPenaltyCostTableINTEL"; + case OpSubgroupAvcMceGetDefaultMediumPenaltyCostTableINTEL: return "OpSubgroupAvcMceGetDefaultMediumPenaltyCostTableINTEL"; + case OpSubgroupAvcMceGetDefaultLowPenaltyCostTableINTEL: return "OpSubgroupAvcMceGetDefaultLowPenaltyCostTableINTEL"; + case OpSubgroupAvcMceSetMotionVectorCostFunctionINTEL: return "OpSubgroupAvcMceSetMotionVectorCostFunctionINTEL"; + case OpSubgroupAvcMceGetDefaultIntraLumaModePenaltyINTEL: return "OpSubgroupAvcMceGetDefaultIntraLumaModePenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultNonDcLumaIntraPenaltyINTEL: return "OpSubgroupAvcMceGetDefaultNonDcLumaIntraPenaltyINTEL"; + case OpSubgroupAvcMceGetDefaultIntraChromaModeBasePenaltyINTEL: return "OpSubgroupAvcMceGetDefaultIntraChromaModeBasePenaltyINTEL"; + case OpSubgroupAvcMceSetAcOnlyHaarINTEL: return "OpSubgroupAvcMceSetAcOnlyHaarINTEL"; + case OpSubgroupAvcMceSetSourceInterlacedFieldPolarityINTEL: return "OpSubgroupAvcMceSetSourceInterlacedFieldPolarityINTEL"; + case OpSubgroupAvcMceSetSingleReferenceInterlacedFieldPolarityINTEL: return "OpSubgroupAvcMceSetSingleReferenceInterlacedFieldPolarityINTEL"; + case OpSubgroupAvcMceSetDualReferenceInterlacedFieldPolaritiesINTEL: return "OpSubgroupAvcMceSetDualReferenceInterlacedFieldPolaritiesINTEL"; + case OpSubgroupAvcMceConvertToImePayloadINTEL: return "OpSubgroupAvcMceConvertToImePayloadINTEL"; + case OpSubgroupAvcMceConvertToImeResultINTEL: return "OpSubgroupAvcMceConvertToImeResultINTEL"; + case OpSubgroupAvcMceConvertToRefPayloadINTEL: return "OpSubgroupAvcMceConvertToRefPayloadINTEL"; + case OpSubgroupAvcMceConvertToRefResultINTEL: return "OpSubgroupAvcMceConvertToRefResultINTEL"; + case OpSubgroupAvcMceConvertToSicPayloadINTEL: return "OpSubgroupAvcMceConvertToSicPayloadINTEL"; + case OpSubgroupAvcMceConvertToSicResultINTEL: return "OpSubgroupAvcMceConvertToSicResultINTEL"; + case OpSubgroupAvcMceGetMotionVectorsINTEL: return "OpSubgroupAvcMceGetMotionVectorsINTEL"; + case OpSubgroupAvcMceGetInterDistortionsINTEL: return "OpSubgroupAvcMceGetInterDistortionsINTEL"; + case OpSubgroupAvcMceGetBestInterDistortionsINTEL: return "OpSubgroupAvcMceGetBestInterDistortionsINTEL"; + case OpSubgroupAvcMceGetInterMajorShapeINTEL: return "OpSubgroupAvcMceGetInterMajorShapeINTEL"; + case OpSubgroupAvcMceGetInterMinorShapeINTEL: return "OpSubgroupAvcMceGetInterMinorShapeINTEL"; + case OpSubgroupAvcMceGetInterDirectionsINTEL: return "OpSubgroupAvcMceGetInterDirectionsINTEL"; + case OpSubgroupAvcMceGetInterMotionVectorCountINTEL: return "OpSubgroupAvcMceGetInterMotionVectorCountINTEL"; + case OpSubgroupAvcMceGetInterReferenceIdsINTEL: return "OpSubgroupAvcMceGetInterReferenceIdsINTEL"; + case OpSubgroupAvcMceGetInterReferenceInterlacedFieldPolaritiesINTEL: return "OpSubgroupAvcMceGetInterReferenceInterlacedFieldPolaritiesINTEL"; + case OpSubgroupAvcImeInitializeINTEL: return "OpSubgroupAvcImeInitializeINTEL"; + case OpSubgroupAvcImeSetSingleReferenceINTEL: return "OpSubgroupAvcImeSetSingleReferenceINTEL"; + case OpSubgroupAvcImeSetDualReferenceINTEL: return "OpSubgroupAvcImeSetDualReferenceINTEL"; + case OpSubgroupAvcImeRefWindowSizeINTEL: return "OpSubgroupAvcImeRefWindowSizeINTEL"; + case OpSubgroupAvcImeAdjustRefOffsetINTEL: return "OpSubgroupAvcImeAdjustRefOffsetINTEL"; + case OpSubgroupAvcImeConvertToMcePayloadINTEL: return "OpSubgroupAvcImeConvertToMcePayloadINTEL"; + case OpSubgroupAvcImeSetMaxMotionVectorCountINTEL: return "OpSubgroupAvcImeSetMaxMotionVectorCountINTEL"; + case OpSubgroupAvcImeSetUnidirectionalMixDisableINTEL: return "OpSubgroupAvcImeSetUnidirectionalMixDisableINTEL"; + case OpSubgroupAvcImeSetEarlySearchTerminationThresholdINTEL: return "OpSubgroupAvcImeSetEarlySearchTerminationThresholdINTEL"; + case OpSubgroupAvcImeSetWeightedSadINTEL: return "OpSubgroupAvcImeSetWeightedSadINTEL"; + case OpSubgroupAvcImeEvaluateWithSingleReferenceINTEL: return "OpSubgroupAvcImeEvaluateWithSingleReferenceINTEL"; + case OpSubgroupAvcImeEvaluateWithDualReferenceINTEL: return "OpSubgroupAvcImeEvaluateWithDualReferenceINTEL"; + case OpSubgroupAvcImeEvaluateWithSingleReferenceStreaminINTEL: return "OpSubgroupAvcImeEvaluateWithSingleReferenceStreaminINTEL"; + case OpSubgroupAvcImeEvaluateWithDualReferenceStreaminINTEL: return "OpSubgroupAvcImeEvaluateWithDualReferenceStreaminINTEL"; + case OpSubgroupAvcImeEvaluateWithSingleReferenceStreamoutINTEL: return "OpSubgroupAvcImeEvaluateWithSingleReferenceStreamoutINTEL"; + case OpSubgroupAvcImeEvaluateWithDualReferenceStreamoutINTEL: return "OpSubgroupAvcImeEvaluateWithDualReferenceStreamoutINTEL"; + case OpSubgroupAvcImeEvaluateWithSingleReferenceStreaminoutINTEL: return "OpSubgroupAvcImeEvaluateWithSingleReferenceStreaminoutINTEL"; + case OpSubgroupAvcImeEvaluateWithDualReferenceStreaminoutINTEL: return "OpSubgroupAvcImeEvaluateWithDualReferenceStreaminoutINTEL"; + case OpSubgroupAvcImeConvertToMceResultINTEL: return "OpSubgroupAvcImeConvertToMceResultINTEL"; + case OpSubgroupAvcImeGetSingleReferenceStreaminINTEL: return "OpSubgroupAvcImeGetSingleReferenceStreaminINTEL"; + case OpSubgroupAvcImeGetDualReferenceStreaminINTEL: return "OpSubgroupAvcImeGetDualReferenceStreaminINTEL"; + case OpSubgroupAvcImeStripSingleReferenceStreamoutINTEL: return "OpSubgroupAvcImeStripSingleReferenceStreamoutINTEL"; + case OpSubgroupAvcImeStripDualReferenceStreamoutINTEL: return "OpSubgroupAvcImeStripDualReferenceStreamoutINTEL"; + case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeMotionVectorsINTEL: return "OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeMotionVectorsINTEL"; + case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeDistortionsINTEL: return "OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeDistortionsINTEL"; + case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeReferenceIdsINTEL: return "OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeReferenceIdsINTEL"; + case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeMotionVectorsINTEL: return "OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeMotionVectorsINTEL"; + case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeDistortionsINTEL: return "OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeDistortionsINTEL"; + case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeReferenceIdsINTEL: return "OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeReferenceIdsINTEL"; + case OpSubgroupAvcImeGetBorderReachedINTEL: return "OpSubgroupAvcImeGetBorderReachedINTEL"; + case OpSubgroupAvcImeGetTruncatedSearchIndicationINTEL: return "OpSubgroupAvcImeGetTruncatedSearchIndicationINTEL"; + case OpSubgroupAvcImeGetUnidirectionalEarlySearchTerminationINTEL: return "OpSubgroupAvcImeGetUnidirectionalEarlySearchTerminationINTEL"; + case OpSubgroupAvcImeGetWeightingPatternMinimumMotionVectorINTEL: return "OpSubgroupAvcImeGetWeightingPatternMinimumMotionVectorINTEL"; + case OpSubgroupAvcImeGetWeightingPatternMinimumDistortionINTEL: return "OpSubgroupAvcImeGetWeightingPatternMinimumDistortionINTEL"; + case OpSubgroupAvcFmeInitializeINTEL: return "OpSubgroupAvcFmeInitializeINTEL"; + case OpSubgroupAvcBmeInitializeINTEL: return "OpSubgroupAvcBmeInitializeINTEL"; + case OpSubgroupAvcRefConvertToMcePayloadINTEL: return "OpSubgroupAvcRefConvertToMcePayloadINTEL"; + case OpSubgroupAvcRefSetBidirectionalMixDisableINTEL: return "OpSubgroupAvcRefSetBidirectionalMixDisableINTEL"; + case OpSubgroupAvcRefSetBilinearFilterEnableINTEL: return "OpSubgroupAvcRefSetBilinearFilterEnableINTEL"; + case OpSubgroupAvcRefEvaluateWithSingleReferenceINTEL: return "OpSubgroupAvcRefEvaluateWithSingleReferenceINTEL"; + case OpSubgroupAvcRefEvaluateWithDualReferenceINTEL: return "OpSubgroupAvcRefEvaluateWithDualReferenceINTEL"; + case OpSubgroupAvcRefEvaluateWithMultiReferenceINTEL: return "OpSubgroupAvcRefEvaluateWithMultiReferenceINTEL"; + case OpSubgroupAvcRefEvaluateWithMultiReferenceInterlacedINTEL: return "OpSubgroupAvcRefEvaluateWithMultiReferenceInterlacedINTEL"; + case OpSubgroupAvcRefConvertToMceResultINTEL: return "OpSubgroupAvcRefConvertToMceResultINTEL"; + case OpSubgroupAvcSicInitializeINTEL: return "OpSubgroupAvcSicInitializeINTEL"; + case OpSubgroupAvcSicConfigureSkcINTEL: return "OpSubgroupAvcSicConfigureSkcINTEL"; + case OpSubgroupAvcSicConfigureIpeLumaINTEL: return "OpSubgroupAvcSicConfigureIpeLumaINTEL"; + case OpSubgroupAvcSicConfigureIpeLumaChromaINTEL: return "OpSubgroupAvcSicConfigureIpeLumaChromaINTEL"; + case OpSubgroupAvcSicGetMotionVectorMaskINTEL: return "OpSubgroupAvcSicGetMotionVectorMaskINTEL"; + case OpSubgroupAvcSicConvertToMcePayloadINTEL: return "OpSubgroupAvcSicConvertToMcePayloadINTEL"; + case OpSubgroupAvcSicSetIntraLumaShapePenaltyINTEL: return "OpSubgroupAvcSicSetIntraLumaShapePenaltyINTEL"; + case OpSubgroupAvcSicSetIntraLumaModeCostFunctionINTEL: return "OpSubgroupAvcSicSetIntraLumaModeCostFunctionINTEL"; + case OpSubgroupAvcSicSetIntraChromaModeCostFunctionINTEL: return "OpSubgroupAvcSicSetIntraChromaModeCostFunctionINTEL"; + case OpSubgroupAvcSicSetBilinearFilterEnableINTEL: return "OpSubgroupAvcSicSetBilinearFilterEnableINTEL"; + case OpSubgroupAvcSicSetSkcForwardTransformEnableINTEL: return "OpSubgroupAvcSicSetSkcForwardTransformEnableINTEL"; + case OpSubgroupAvcSicSetBlockBasedRawSkipSadINTEL: return "OpSubgroupAvcSicSetBlockBasedRawSkipSadINTEL"; + case OpSubgroupAvcSicEvaluateIpeINTEL: return "OpSubgroupAvcSicEvaluateIpeINTEL"; + case OpSubgroupAvcSicEvaluateWithSingleReferenceINTEL: return "OpSubgroupAvcSicEvaluateWithSingleReferenceINTEL"; + case OpSubgroupAvcSicEvaluateWithDualReferenceINTEL: return "OpSubgroupAvcSicEvaluateWithDualReferenceINTEL"; + case OpSubgroupAvcSicEvaluateWithMultiReferenceINTEL: return "OpSubgroupAvcSicEvaluateWithMultiReferenceINTEL"; + case OpSubgroupAvcSicEvaluateWithMultiReferenceInterlacedINTEL: return "OpSubgroupAvcSicEvaluateWithMultiReferenceInterlacedINTEL"; + case OpSubgroupAvcSicConvertToMceResultINTEL: return "OpSubgroupAvcSicConvertToMceResultINTEL"; + case OpSubgroupAvcSicGetIpeLumaShapeINTEL: return "OpSubgroupAvcSicGetIpeLumaShapeINTEL"; + case OpSubgroupAvcSicGetBestIpeLumaDistortionINTEL: return "OpSubgroupAvcSicGetBestIpeLumaDistortionINTEL"; + case OpSubgroupAvcSicGetBestIpeChromaDistortionINTEL: return "OpSubgroupAvcSicGetBestIpeChromaDistortionINTEL"; + case OpSubgroupAvcSicGetPackedIpeLumaModesINTEL: return "OpSubgroupAvcSicGetPackedIpeLumaModesINTEL"; + case OpSubgroupAvcSicGetIpeChromaModeINTEL: return "OpSubgroupAvcSicGetIpeChromaModeINTEL"; + case OpSubgroupAvcSicGetPackedSkcLumaCountThresholdINTEL: return "OpSubgroupAvcSicGetPackedSkcLumaCountThresholdINTEL"; + case OpSubgroupAvcSicGetPackedSkcLumaSumThresholdINTEL: return "OpSubgroupAvcSicGetPackedSkcLumaSumThresholdINTEL"; + case OpSubgroupAvcSicGetInterRawSadsINTEL: return "OpSubgroupAvcSicGetInterRawSadsINTEL"; + case OpVariableLengthArrayINTEL: return "OpVariableLengthArrayINTEL"; + case OpSaveMemoryINTEL: return "OpSaveMemoryINTEL"; + case OpRestoreMemoryINTEL: return "OpRestoreMemoryINTEL"; + case OpArbitraryFloatSinCosPiALTERA: return "OpArbitraryFloatSinCosPiALTERA"; + case OpArbitraryFloatCastALTERA: return "OpArbitraryFloatCastALTERA"; + case OpArbitraryFloatCastFromIntALTERA: return "OpArbitraryFloatCastFromIntALTERA"; + case OpArbitraryFloatCastToIntALTERA: return "OpArbitraryFloatCastToIntALTERA"; + case OpArbitraryFloatAddALTERA: return "OpArbitraryFloatAddALTERA"; + case OpArbitraryFloatSubALTERA: return "OpArbitraryFloatSubALTERA"; + case OpArbitraryFloatMulALTERA: return "OpArbitraryFloatMulALTERA"; + case OpArbitraryFloatDivALTERA: return "OpArbitraryFloatDivALTERA"; + case OpArbitraryFloatGTALTERA: return "OpArbitraryFloatGTALTERA"; + case OpArbitraryFloatGEALTERA: return "OpArbitraryFloatGEALTERA"; + case OpArbitraryFloatLTALTERA: return "OpArbitraryFloatLTALTERA"; + case OpArbitraryFloatLEALTERA: return "OpArbitraryFloatLEALTERA"; + case OpArbitraryFloatEQALTERA: return "OpArbitraryFloatEQALTERA"; + case OpArbitraryFloatRecipALTERA: return "OpArbitraryFloatRecipALTERA"; + case OpArbitraryFloatRSqrtALTERA: return "OpArbitraryFloatRSqrtALTERA"; + case OpArbitraryFloatCbrtALTERA: return "OpArbitraryFloatCbrtALTERA"; + case OpArbitraryFloatHypotALTERA: return "OpArbitraryFloatHypotALTERA"; + case OpArbitraryFloatSqrtALTERA: return "OpArbitraryFloatSqrtALTERA"; + case OpArbitraryFloatLogINTEL: return "OpArbitraryFloatLogINTEL"; + case OpArbitraryFloatLog2INTEL: return "OpArbitraryFloatLog2INTEL"; + case OpArbitraryFloatLog10INTEL: return "OpArbitraryFloatLog10INTEL"; + case OpArbitraryFloatLog1pINTEL: return "OpArbitraryFloatLog1pINTEL"; + case OpArbitraryFloatExpINTEL: return "OpArbitraryFloatExpINTEL"; + case OpArbitraryFloatExp2INTEL: return "OpArbitraryFloatExp2INTEL"; + case OpArbitraryFloatExp10INTEL: return "OpArbitraryFloatExp10INTEL"; + case OpArbitraryFloatExpm1INTEL: return "OpArbitraryFloatExpm1INTEL"; + case OpArbitraryFloatSinINTEL: return "OpArbitraryFloatSinINTEL"; + case OpArbitraryFloatCosINTEL: return "OpArbitraryFloatCosINTEL"; + case OpArbitraryFloatSinCosINTEL: return "OpArbitraryFloatSinCosINTEL"; + case OpArbitraryFloatSinPiINTEL: return "OpArbitraryFloatSinPiINTEL"; + case OpArbitraryFloatCosPiINTEL: return "OpArbitraryFloatCosPiINTEL"; + case OpArbitraryFloatASinINTEL: return "OpArbitraryFloatASinINTEL"; + case OpArbitraryFloatASinPiINTEL: return "OpArbitraryFloatASinPiINTEL"; + case OpArbitraryFloatACosINTEL: return "OpArbitraryFloatACosINTEL"; + case OpArbitraryFloatACosPiINTEL: return "OpArbitraryFloatACosPiINTEL"; + case OpArbitraryFloatATanINTEL: return "OpArbitraryFloatATanINTEL"; + case OpArbitraryFloatATanPiINTEL: return "OpArbitraryFloatATanPiINTEL"; + case OpArbitraryFloatATan2INTEL: return "OpArbitraryFloatATan2INTEL"; + case OpArbitraryFloatPowINTEL: return "OpArbitraryFloatPowINTEL"; + case OpArbitraryFloatPowRINTEL: return "OpArbitraryFloatPowRINTEL"; + case OpArbitraryFloatPowNINTEL: return "OpArbitraryFloatPowNINTEL"; + case OpLoopControlINTEL: return "OpLoopControlINTEL"; + case OpAliasDomainDeclINTEL: return "OpAliasDomainDeclINTEL"; + case OpAliasScopeDeclINTEL: return "OpAliasScopeDeclINTEL"; + case OpAliasScopeListDeclINTEL: return "OpAliasScopeListDeclINTEL"; + case OpFixedSqrtALTERA: return "OpFixedSqrtALTERA"; + case OpFixedRecipALTERA: return "OpFixedRecipALTERA"; + case OpFixedRsqrtALTERA: return "OpFixedRsqrtALTERA"; + case OpFixedSinALTERA: return "OpFixedSinALTERA"; + case OpFixedCosALTERA: return "OpFixedCosALTERA"; + case OpFixedSinCosALTERA: return "OpFixedSinCosALTERA"; + case OpFixedSinPiALTERA: return "OpFixedSinPiALTERA"; + case OpFixedCosPiALTERA: return "OpFixedCosPiALTERA"; + case OpFixedSinCosPiALTERA: return "OpFixedSinCosPiALTERA"; + case OpFixedLogALTERA: return "OpFixedLogALTERA"; + case OpFixedExpALTERA: return "OpFixedExpALTERA"; + case OpPtrCastToCrossWorkgroupALTERA: return "OpPtrCastToCrossWorkgroupALTERA"; + case OpCrossWorkgroupCastToPtrALTERA: return "OpCrossWorkgroupCastToPtrALTERA"; + case OpReadPipeBlockingALTERA: return "OpReadPipeBlockingALTERA"; + case OpWritePipeBlockingALTERA: return "OpWritePipeBlockingALTERA"; + case OpFPGARegALTERA: return "OpFPGARegALTERA"; + case OpRayQueryGetRayTMinKHR: return "OpRayQueryGetRayTMinKHR"; + case OpRayQueryGetRayFlagsKHR: return "OpRayQueryGetRayFlagsKHR"; + case OpRayQueryGetIntersectionTKHR: return "OpRayQueryGetIntersectionTKHR"; + case OpRayQueryGetIntersectionInstanceCustomIndexKHR: return "OpRayQueryGetIntersectionInstanceCustomIndexKHR"; + case OpRayQueryGetIntersectionInstanceIdKHR: return "OpRayQueryGetIntersectionInstanceIdKHR"; + case OpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR: return "OpRayQueryGetIntersectionInstanceShaderBindingTableRecordOffsetKHR"; + case OpRayQueryGetIntersectionGeometryIndexKHR: return "OpRayQueryGetIntersectionGeometryIndexKHR"; + case OpRayQueryGetIntersectionPrimitiveIndexKHR: return "OpRayQueryGetIntersectionPrimitiveIndexKHR"; + case OpRayQueryGetIntersectionBarycentricsKHR: return "OpRayQueryGetIntersectionBarycentricsKHR"; + case OpRayQueryGetIntersectionFrontFaceKHR: return "OpRayQueryGetIntersectionFrontFaceKHR"; + case OpRayQueryGetIntersectionCandidateAABBOpaqueKHR: return "OpRayQueryGetIntersectionCandidateAABBOpaqueKHR"; + case OpRayQueryGetIntersectionObjectRayDirectionKHR: return "OpRayQueryGetIntersectionObjectRayDirectionKHR"; + case OpRayQueryGetIntersectionObjectRayOriginKHR: return "OpRayQueryGetIntersectionObjectRayOriginKHR"; + case OpRayQueryGetWorldRayDirectionKHR: return "OpRayQueryGetWorldRayDirectionKHR"; + case OpRayQueryGetWorldRayOriginKHR: return "OpRayQueryGetWorldRayOriginKHR"; + case OpRayQueryGetIntersectionObjectToWorldKHR: return "OpRayQueryGetIntersectionObjectToWorldKHR"; + case OpRayQueryGetIntersectionWorldToObjectKHR: return "OpRayQueryGetIntersectionWorldToObjectKHR"; + case OpAtomicFAddEXT: return "OpAtomicFAddEXT"; + case OpTypeBufferSurfaceINTEL: return "OpTypeBufferSurfaceINTEL"; + case OpTypeStructContinuedINTEL: return "OpTypeStructContinuedINTEL"; + case OpConstantCompositeContinuedINTEL: return "OpConstantCompositeContinuedINTEL"; + case OpSpecConstantCompositeContinuedINTEL: return "OpSpecConstantCompositeContinuedINTEL"; + case OpCompositeConstructContinuedINTEL: return "OpCompositeConstructContinuedINTEL"; + case OpConvertFToBF16INTEL: return "OpConvertFToBF16INTEL"; + case OpConvertBF16ToFINTEL: return "OpConvertBF16ToFINTEL"; + case OpControlBarrierArriveINTEL: return "OpControlBarrierArriveINTEL"; + case OpControlBarrierWaitINTEL: return "OpControlBarrierWaitINTEL"; + case OpArithmeticFenceEXT: return "OpArithmeticFenceEXT"; + case OpTaskSequenceCreateALTERA: return "OpTaskSequenceCreateALTERA"; + case OpTaskSequenceAsyncALTERA: return "OpTaskSequenceAsyncALTERA"; + case OpTaskSequenceGetALTERA: return "OpTaskSequenceGetALTERA"; + case OpTaskSequenceReleaseALTERA: return "OpTaskSequenceReleaseALTERA"; + case OpTypeTaskSequenceALTERA: return "OpTypeTaskSequenceALTERA"; + case OpSubgroupBlockPrefetchINTEL: return "OpSubgroupBlockPrefetchINTEL"; + case OpSubgroup2DBlockLoadINTEL: return "OpSubgroup2DBlockLoadINTEL"; + case OpSubgroup2DBlockLoadTransformINTEL: return "OpSubgroup2DBlockLoadTransformINTEL"; + case OpSubgroup2DBlockLoadTransposeINTEL: return "OpSubgroup2DBlockLoadTransposeINTEL"; + case OpSubgroup2DBlockPrefetchINTEL: return "OpSubgroup2DBlockPrefetchINTEL"; + case OpSubgroup2DBlockStoreINTEL: return "OpSubgroup2DBlockStoreINTEL"; + case OpSubgroupMatrixMultiplyAccumulateINTEL: return "OpSubgroupMatrixMultiplyAccumulateINTEL"; + case OpBitwiseFunctionINTEL: return "OpBitwiseFunctionINTEL"; + case OpUntypedVariableLengthArrayINTEL: return "OpUntypedVariableLengthArrayINTEL"; + case OpConditionalExtensionINTEL: return "OpConditionalExtensionINTEL"; + case OpConditionalEntryPointINTEL: return "OpConditionalEntryPointINTEL"; + case OpConditionalCapabilityINTEL: return "OpConditionalCapabilityINTEL"; + case OpSpecConstantTargetINTEL: return "OpSpecConstantTargetINTEL"; + case OpSpecConstantArchitectureINTEL: return "OpSpecConstantArchitectureINTEL"; + case OpSpecConstantCapabilitiesINTEL: return "OpSpecConstantCapabilitiesINTEL"; + case OpConditionalCopyObjectINTEL: return "OpConditionalCopyObjectINTEL"; + case OpGroupIMulKHR: return "OpGroupIMulKHR"; + case OpGroupFMulKHR: return "OpGroupFMulKHR"; + case OpGroupBitwiseAndKHR: return "OpGroupBitwiseAndKHR"; + case OpGroupBitwiseOrKHR: return "OpGroupBitwiseOrKHR"; + case OpGroupBitwiseXorKHR: return "OpGroupBitwiseXorKHR"; + case OpGroupLogicalAndKHR: return "OpGroupLogicalAndKHR"; + case OpGroupLogicalOrKHR: return "OpGroupLogicalOrKHR"; + case OpGroupLogicalXorKHR: return "OpGroupLogicalXorKHR"; + case OpRoundFToTF32INTEL: return "OpRoundFToTF32INTEL"; + case OpMaskedGatherINTEL: return "OpMaskedGatherINTEL"; + case OpMaskedScatterINTEL: return "OpMaskedScatterINTEL"; + case OpConvertHandleToImageINTEL: return "OpConvertHandleToImageINTEL"; + case OpConvertHandleToSamplerINTEL: return "OpConvertHandleToSamplerINTEL"; + case OpConvertHandleToSampledImageINTEL: return "OpConvertHandleToSampledImageINTEL"; + case OpFDot2MixAcc32VALVE: return "OpFDot2MixAcc32VALVE"; + case OpFDot2MixAcc16VALVE: return "OpFDot2MixAcc16VALVE"; + case OpFDot4MixAcc32VALVE: return "OpFDot4MixAcc32VALVE"; + default: return "Unknown"; + } +} + #endif /* SPV_ENABLE_UTILITY_CODE */ -// Overload operator| for mask bit combining +// Overload bitwise operators for mask bit combining inline ImageOperandsMask operator|(ImageOperandsMask a, ImageOperandsMask b) { return ImageOperandsMask(unsigned(a) | unsigned(b)); } +inline ImageOperandsMask operator&(ImageOperandsMask a, ImageOperandsMask b) { return ImageOperandsMask(unsigned(a) & unsigned(b)); } +inline ImageOperandsMask operator^(ImageOperandsMask a, ImageOperandsMask b) { return ImageOperandsMask(unsigned(a) ^ unsigned(b)); } +inline ImageOperandsMask operator~(ImageOperandsMask a) { return ImageOperandsMask(~unsigned(a)); } inline FPFastMathModeMask operator|(FPFastMathModeMask a, FPFastMathModeMask b) { return FPFastMathModeMask(unsigned(a) | unsigned(b)); } +inline FPFastMathModeMask operator&(FPFastMathModeMask a, FPFastMathModeMask b) { return FPFastMathModeMask(unsigned(a) & unsigned(b)); } +inline FPFastMathModeMask operator^(FPFastMathModeMask a, FPFastMathModeMask b) { return FPFastMathModeMask(unsigned(a) ^ unsigned(b)); } +inline FPFastMathModeMask operator~(FPFastMathModeMask a) { return FPFastMathModeMask(~unsigned(a)); } inline SelectionControlMask operator|(SelectionControlMask a, SelectionControlMask b) { return SelectionControlMask(unsigned(a) | unsigned(b)); } +inline SelectionControlMask operator&(SelectionControlMask a, SelectionControlMask b) { return SelectionControlMask(unsigned(a) & unsigned(b)); } +inline SelectionControlMask operator^(SelectionControlMask a, SelectionControlMask b) { return SelectionControlMask(unsigned(a) ^ unsigned(b)); } +inline SelectionControlMask operator~(SelectionControlMask a) { return SelectionControlMask(~unsigned(a)); } inline LoopControlMask operator|(LoopControlMask a, LoopControlMask b) { return LoopControlMask(unsigned(a) | unsigned(b)); } +inline LoopControlMask operator&(LoopControlMask a, LoopControlMask b) { return LoopControlMask(unsigned(a) & unsigned(b)); } +inline LoopControlMask operator^(LoopControlMask a, LoopControlMask b) { return LoopControlMask(unsigned(a) ^ unsigned(b)); } +inline LoopControlMask operator~(LoopControlMask a) { return LoopControlMask(~unsigned(a)); } inline FunctionControlMask operator|(FunctionControlMask a, FunctionControlMask b) { return FunctionControlMask(unsigned(a) | unsigned(b)); } +inline FunctionControlMask operator&(FunctionControlMask a, FunctionControlMask b) { return FunctionControlMask(unsigned(a) & unsigned(b)); } +inline FunctionControlMask operator^(FunctionControlMask a, FunctionControlMask b) { return FunctionControlMask(unsigned(a) ^ unsigned(b)); } +inline FunctionControlMask operator~(FunctionControlMask a) { return FunctionControlMask(~unsigned(a)); } inline MemorySemanticsMask operator|(MemorySemanticsMask a, MemorySemanticsMask b) { return MemorySemanticsMask(unsigned(a) | unsigned(b)); } +inline MemorySemanticsMask operator&(MemorySemanticsMask a, MemorySemanticsMask b) { return MemorySemanticsMask(unsigned(a) & unsigned(b)); } +inline MemorySemanticsMask operator^(MemorySemanticsMask a, MemorySemanticsMask b) { return MemorySemanticsMask(unsigned(a) ^ unsigned(b)); } +inline MemorySemanticsMask operator~(MemorySemanticsMask a) { return MemorySemanticsMask(~unsigned(a)); } inline MemoryAccessMask operator|(MemoryAccessMask a, MemoryAccessMask b) { return MemoryAccessMask(unsigned(a) | unsigned(b)); } +inline MemoryAccessMask operator&(MemoryAccessMask a, MemoryAccessMask b) { return MemoryAccessMask(unsigned(a) & unsigned(b)); } +inline MemoryAccessMask operator^(MemoryAccessMask a, MemoryAccessMask b) { return MemoryAccessMask(unsigned(a) ^ unsigned(b)); } +inline MemoryAccessMask operator~(MemoryAccessMask a) { return MemoryAccessMask(~unsigned(a)); } inline KernelProfilingInfoMask operator|(KernelProfilingInfoMask a, KernelProfilingInfoMask b) { return KernelProfilingInfoMask(unsigned(a) | unsigned(b)); } +inline KernelProfilingInfoMask operator&(KernelProfilingInfoMask a, KernelProfilingInfoMask b) { return KernelProfilingInfoMask(unsigned(a) & unsigned(b)); } +inline KernelProfilingInfoMask operator^(KernelProfilingInfoMask a, KernelProfilingInfoMask b) { return KernelProfilingInfoMask(unsigned(a) ^ unsigned(b)); } +inline KernelProfilingInfoMask operator~(KernelProfilingInfoMask a) { return KernelProfilingInfoMask(~unsigned(a)); } inline RayFlagsMask operator|(RayFlagsMask a, RayFlagsMask b) { return RayFlagsMask(unsigned(a) | unsigned(b)); } +inline RayFlagsMask operator&(RayFlagsMask a, RayFlagsMask b) { return RayFlagsMask(unsigned(a) & unsigned(b)); } +inline RayFlagsMask operator^(RayFlagsMask a, RayFlagsMask b) { return RayFlagsMask(unsigned(a) ^ unsigned(b)); } +inline RayFlagsMask operator~(RayFlagsMask a) { return RayFlagsMask(~unsigned(a)); } inline FragmentShadingRateMask operator|(FragmentShadingRateMask a, FragmentShadingRateMask b) { return FragmentShadingRateMask(unsigned(a) | unsigned(b)); } +inline FragmentShadingRateMask operator&(FragmentShadingRateMask a, FragmentShadingRateMask b) { return FragmentShadingRateMask(unsigned(a) & unsigned(b)); } +inline FragmentShadingRateMask operator^(FragmentShadingRateMask a, FragmentShadingRateMask b) { return FragmentShadingRateMask(unsigned(a) ^ unsigned(b)); } +inline FragmentShadingRateMask operator~(FragmentShadingRateMask a) { return FragmentShadingRateMask(~unsigned(a)); } +inline CooperativeMatrixOperandsMask operator|(CooperativeMatrixOperandsMask a, CooperativeMatrixOperandsMask b) { return CooperativeMatrixOperandsMask(unsigned(a) | unsigned(b)); } +inline CooperativeMatrixOperandsMask operator&(CooperativeMatrixOperandsMask a, CooperativeMatrixOperandsMask b) { return CooperativeMatrixOperandsMask(unsigned(a) & unsigned(b)); } +inline CooperativeMatrixOperandsMask operator^(CooperativeMatrixOperandsMask a, CooperativeMatrixOperandsMask b) { return CooperativeMatrixOperandsMask(unsigned(a) ^ unsigned(b)); } +inline CooperativeMatrixOperandsMask operator~(CooperativeMatrixOperandsMask a) { return CooperativeMatrixOperandsMask(~unsigned(a)); } +inline CooperativeMatrixReduceMask operator|(CooperativeMatrixReduceMask a, CooperativeMatrixReduceMask b) { return CooperativeMatrixReduceMask(unsigned(a) | unsigned(b)); } +inline CooperativeMatrixReduceMask operator&(CooperativeMatrixReduceMask a, CooperativeMatrixReduceMask b) { return CooperativeMatrixReduceMask(unsigned(a) & unsigned(b)); } +inline CooperativeMatrixReduceMask operator^(CooperativeMatrixReduceMask a, CooperativeMatrixReduceMask b) { return CooperativeMatrixReduceMask(unsigned(a) ^ unsigned(b)); } +inline CooperativeMatrixReduceMask operator~(CooperativeMatrixReduceMask a) { return CooperativeMatrixReduceMask(~unsigned(a)); } +inline TensorAddressingOperandsMask operator|(TensorAddressingOperandsMask a, TensorAddressingOperandsMask b) { return TensorAddressingOperandsMask(unsigned(a) | unsigned(b)); } +inline TensorAddressingOperandsMask operator&(TensorAddressingOperandsMask a, TensorAddressingOperandsMask b) { return TensorAddressingOperandsMask(unsigned(a) & unsigned(b)); } +inline TensorAddressingOperandsMask operator^(TensorAddressingOperandsMask a, TensorAddressingOperandsMask b) { return TensorAddressingOperandsMask(unsigned(a) ^ unsigned(b)); } +inline TensorAddressingOperandsMask operator~(TensorAddressingOperandsMask a) { return TensorAddressingOperandsMask(~unsigned(a)); } +inline TensorOperandsMask operator|(TensorOperandsMask a, TensorOperandsMask b) { return TensorOperandsMask(unsigned(a) | unsigned(b)); } +inline TensorOperandsMask operator&(TensorOperandsMask a, TensorOperandsMask b) { return TensorOperandsMask(unsigned(a) & unsigned(b)); } +inline TensorOperandsMask operator^(TensorOperandsMask a, TensorOperandsMask b) { return TensorOperandsMask(unsigned(a) ^ unsigned(b)); } +inline TensorOperandsMask operator~(TensorOperandsMask a) { return TensorOperandsMask(~unsigned(a)); } +inline MatrixMultiplyAccumulateOperandsMask operator|(MatrixMultiplyAccumulateOperandsMask a, MatrixMultiplyAccumulateOperandsMask b) { return MatrixMultiplyAccumulateOperandsMask(unsigned(a) | unsigned(b)); } +inline MatrixMultiplyAccumulateOperandsMask operator&(MatrixMultiplyAccumulateOperandsMask a, MatrixMultiplyAccumulateOperandsMask b) { return MatrixMultiplyAccumulateOperandsMask(unsigned(a) & unsigned(b)); } +inline MatrixMultiplyAccumulateOperandsMask operator^(MatrixMultiplyAccumulateOperandsMask a, MatrixMultiplyAccumulateOperandsMask b) { return MatrixMultiplyAccumulateOperandsMask(unsigned(a) ^ unsigned(b)); } +inline MatrixMultiplyAccumulateOperandsMask operator~(MatrixMultiplyAccumulateOperandsMask a) { return MatrixMultiplyAccumulateOperandsMask(~unsigned(a)); } +inline RawAccessChainOperandsMask operator|(RawAccessChainOperandsMask a, RawAccessChainOperandsMask b) { return RawAccessChainOperandsMask(unsigned(a) | unsigned(b)); } +inline RawAccessChainOperandsMask operator&(RawAccessChainOperandsMask a, RawAccessChainOperandsMask b) { return RawAccessChainOperandsMask(unsigned(a) & unsigned(b)); } +inline RawAccessChainOperandsMask operator^(RawAccessChainOperandsMask a, RawAccessChainOperandsMask b) { return RawAccessChainOperandsMask(unsigned(a) ^ unsigned(b)); } +inline RawAccessChainOperandsMask operator~(RawAccessChainOperandsMask a) { return RawAccessChainOperandsMask(~unsigned(a)); } } // end namespace spv diff --git a/src/libraries/spirv_cross/spirv_cfg.cpp b/src/libraries/spirv_cross/spirv_cfg.cpp index 932994798..ae928a03a 100644 --- a/src/libraries/spirv_cross/spirv_cfg.cpp +++ b/src/libraries/spirv_cross/spirv_cfg.cpp @@ -59,20 +59,26 @@ void CFG::build_immediate_dominators() for (auto i = post_order.size(); i; i--) { uint32_t block = post_order[i - 1]; - auto &pred = preceding_edges[block]; - if (pred.empty()) // This is for the entry block, but we've already set up the dominators. - continue; - for (auto &edge : pred) + const auto resolve_preds = [&](const SmallVector &pred) { - if (immediate_dominators[block]) + if (pred.empty()) // This is for the entry block, but we've already set up the dominators. + return; + + for (auto &edge : pred) { - assert(immediate_dominators[edge]); - immediate_dominators[block] = find_common_dominator(immediate_dominators[block], edge); + if (immediate_dominators[block]) + { + assert(immediate_dominators[edge]); + immediate_dominators[block] = find_common_dominator(immediate_dominators[block], edge); + } + else + immediate_dominators[block] = edge; } - else - immediate_dominators[block] = edge; - } + }; + + resolve_preds(preceding_edges[block]); + resolve_preds(virtual_dominance_preceding_edges[block]); } } @@ -81,31 +87,105 @@ bool CFG::is_back_edge(uint32_t to) const // We have a back edge if the visit order is set with the temporary magic value 0. // Crossing edges will have already been recorded with a visit order. auto itr = visit_order.find(to); - return itr != end(visit_order) && itr->second.get() == 0; + return itr != end(visit_order) && itr->second.visited_branches && !itr->second.visited_resolve; } -bool CFG::has_visited_forward_edge(uint32_t to) const +bool CFG::has_visited_branch(uint32_t to) const { - // If > 0, we have visited the edge already, and this is not a back edge branch. auto itr = visit_order.find(to); - return itr != end(visit_order) && itr->second.get() > 0; + return itr != end(visit_order) && itr->second.visited_branches; } -bool CFG::post_order_visit(uint32_t block_id) +void CFG::post_order_visit_entry(uint32_t block) { - // If we have already branched to this block (back edge), stop recursion. - // If our branches are back-edges, we do not record them. - // We have to record crossing edges however. - if (has_visited_forward_edge(block_id)) - return true; - else if (is_back_edge(block_id)) - return false; + visit_stack.push_back(block); - // Block back-edges from recursively revisiting ourselves. - visit_order[block_id].get() = 0; + while (!visit_stack.empty()) + { + bool keep_iterating; + do + { + // Reverse the order to allow for stack-like behavior and preserves the visit order from recursive algorithm. + // Traverse depth first. + uint32_t to_visit = visit_stack.back(); + last_visited_size = visit_stack.size(); + post_order_visit_branches(to_visit); + keep_iterating = last_visited_size != visit_stack.size(); + if (keep_iterating) + std::reverse(visit_stack.begin() + last_visited_size, visit_stack.end()); + } while (keep_iterating); + // We've reached the end of some tree leaf. Resolve the stack. + // Any node which has been visited for real can be popped now. + while (!visit_stack.empty() && visit_order[visit_stack.back()].visited_branches) + { + post_order_visit_resolve(visit_stack.back()); + visit_stack.pop_back(); + } + } +} + +void CFG::visit_branch(uint32_t block_id) +{ + // Prune obvious duplicates. + if (std::find(visit_stack.begin() + last_visited_size, visit_stack.end(), block_id) == visit_stack.end() && + !has_visited_branch(block_id)) + { + visit_stack.push_back(block_id); + } +} + +void CFG::post_order_visit_branches(uint32_t block_id) +{ auto &block = compiler.get(block_id); + auto &visit = visit_order[block_id]; + if (visit.visited_branches) + return; + visit.visited_branches = true; + + if (block.merge == SPIRBlock::MergeLoop) + visit_branch(block.merge_block); + else if (block.merge == SPIRBlock::MergeSelection) + visit_branch(block.next_block); + + // First visit our branch targets. + switch (block.terminator) + { + case SPIRBlock::Direct: + visit_branch(block.next_block); + break; + + case SPIRBlock::Select: + visit_branch(block.true_block); + visit_branch(block.false_block); + break; + + case SPIRBlock::MultiSelect: + { + const auto &cases = compiler.get_case_list(block); + for (const auto &target : cases) + visit_branch(target.block); + if (block.default_block) + visit_branch(block.default_block); + break; + } + + default: + break; + } +} + +void CFG::post_order_visit_resolve(uint32_t block_id) +{ + auto &block = compiler.get(block_id); + + auto &visit_block = visit_order[block_id]; + assert(visit_block.visited_branches); + auto &visited = visit_order[block_id].visited_resolve; + if (visited) + return; + // If this is a loop header, add an implied branch to the merge target. // This is needed to avoid annoying cases with do { ... } while(false) loops often generated by inliners. // To the CFG, this is linear control flow, but we risk picking the do/while scope as our dominating block. @@ -116,21 +196,28 @@ bool CFG::post_order_visit(uint32_t block_id) // is lower than inside the loop, which is going to be key for some traversal algorithms like post-dominance analysis. // For selection constructs true/false blocks will end up visiting the merge block directly and it works out fine, // but for loops, only the header might end up actually branching to merge block. - if (block.merge == SPIRBlock::MergeLoop && post_order_visit(block.merge_block)) + if (block.merge == SPIRBlock::MergeLoop && !is_back_edge(block.merge_block)) add_branch(block_id, block.merge_block); + // Similar case as do/while loops, but expressed in a different form. + // if (true) { foo = 1; } else { return/unreachable/kill/blah; } access(foo); + // Only consider this branch when computing dominance to avoid breaking other analysis like + // parameter preservation. + if (block.merge == SPIRBlock::MergeSelection && !is_back_edge(block.next_block)) + add_virtual_dominance_branch(block_id, block.next_block); + // First visit our branch targets. switch (block.terminator) { case SPIRBlock::Direct: - if (post_order_visit(block.next_block)) + if (!is_back_edge(block.next_block)) add_branch(block_id, block.next_block); break; case SPIRBlock::Select: - if (post_order_visit(block.true_block)) + if (!is_back_edge(block.true_block)) add_branch(block_id, block.true_block); - if (post_order_visit(block.false_block)) + if (!is_back_edge(block.false_block)) add_branch(block_id, block.false_block); break; @@ -139,10 +226,10 @@ bool CFG::post_order_visit(uint32_t block_id) const auto &cases = compiler.get_case_list(block); for (const auto &target : cases) { - if (post_order_visit(target.block)) + if (!is_back_edge(target.block)) add_branch(block_id, target.block); } - if (block.default_block && post_order_visit(block.default_block)) + if (block.default_block && !is_back_edge(block.default_block)) add_branch(block_id, block.default_block); break; } @@ -157,7 +244,7 @@ bool CFG::post_order_visit(uint32_t block_id) // We can use the variable without a Phi since there is only one possible parent here. // However, in this case, we need to hoist out the inner variable to outside the branch. // Use same strategy as loops. - if (block.merge == SPIRBlock::MergeSelection && post_order_visit(block.next_block)) + if (block.merge == SPIRBlock::MergeSelection && !is_back_edge(block.next_block)) { // If there is only one preceding edge to the merge block and it's not ourselves, we need a fixup. // Add a fake branch so any dominator in either the if (), or else () block, or a lone case statement @@ -201,10 +288,9 @@ bool CFG::post_order_visit(uint32_t block_id) } } - // Then visit ourselves. Start counting at one, to let 0 be a magic value for testing back vs. crossing edges. - visit_order[block_id].get() = ++visit_count; + visited = true; + visit_block.order = ++visit_count; post_order.push_back(block_id); - return true; } void CFG::build_post_order_visit_order() @@ -213,20 +299,29 @@ void CFG::build_post_order_visit_order() visit_count = 0; visit_order.clear(); post_order.clear(); - post_order_visit(block); + post_order_visit_entry(block); +} + +static void add_unique(SmallVector &l, uint32_t value) +{ + auto itr = find(begin(l), end(l), value); + if (itr == end(l)) + l.push_back(value); } void CFG::add_branch(uint32_t from, uint32_t to) { - const auto add_unique = [](SmallVector &l, uint32_t value) { - auto itr = find(begin(l), end(l), value); - if (itr == end(l)) - l.push_back(value); - }; + assert(from && to); add_unique(preceding_edges[to], from); add_unique(succeeding_edges[from], to); } +void CFG::add_virtual_dominance_branch(uint32_t from, uint32_t to) +{ + assert(from && to); + add_unique(virtual_dominance_preceding_edges[to], from); +} + uint32_t CFG::find_loop_dominator(uint32_t block_id) const { while (block_id != SPIRBlock::NoDominator) diff --git a/src/libraries/spirv_cross/spirv_cfg.hpp b/src/libraries/spirv_cross/spirv_cfg.hpp index 1d85fe0a9..9e6141bbe 100644 --- a/src/libraries/spirv_cross/spirv_cfg.hpp +++ b/src/libraries/spirv_cross/spirv_cfg.hpp @@ -68,7 +68,7 @@ public: { auto itr = visit_order.find(block); assert(itr != std::end(visit_order)); - int v = itr->second.get(); + int v = itr->second.order; assert(v > 0); return uint32_t(v); } @@ -114,22 +114,15 @@ public: private: struct VisitOrder { - int &get() - { - return v; - } - - const int &get() const - { - return v; - } - - int v = -1; + int order = -1; + bool visited_resolve = false; + bool visited_branches = false; }; Compiler &compiler; const SPIRFunction &func; std::unordered_map> preceding_edges; + std::unordered_map> virtual_dominance_preceding_edges; std::unordered_map> succeeding_edges; std::unordered_map immediate_dominators; std::unordered_map visit_order; @@ -137,13 +130,20 @@ private: SmallVector empty_vector; void add_branch(uint32_t from, uint32_t to); + void add_virtual_dominance_branch(uint32_t from, uint32_t to); void build_post_order_visit_order(); void build_immediate_dominators(); - bool post_order_visit(uint32_t block); + void post_order_visit_branches(uint32_t block); + void post_order_visit_resolve(uint32_t block); + void post_order_visit_entry(uint32_t block); uint32_t visit_count = 0; bool is_back_edge(uint32_t to) const; - bool has_visited_forward_edge(uint32_t to) const; + bool has_visited_branch(uint32_t to) const; + void visit_branch(uint32_t block_id); + + SmallVector visit_stack; + size_t last_visited_size = 0; }; class DominatorBuilder diff --git a/src/libraries/spirv_cross/spirv_common.hpp b/src/libraries/spirv_cross/spirv_common.hpp index 7fce7ae7e..b003d4db1 100644 --- a/src/libraries/spirv_cross/spirv_common.hpp +++ b/src/libraries/spirv_cross/spirv_common.hpp @@ -27,8 +27,17 @@ #ifndef SPV_ENABLE_UTILITY_CODE #define SPV_ENABLE_UTILITY_CODE #endif -#include "spirv.hpp" +// Pragmatic hack to avoid symbol conflicts when including both hpp11 and hpp headers in same translation unit. +// This is an unfortunate SPIRV-Headers issue that we cannot easily deal with ourselves. +#ifdef SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#define spv SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#define SPIRV_CROSS_SPV_HEADER_NAMESPACE SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#else +#define SPIRV_CROSS_SPV_HEADER_NAMESPACE spv +#endif + +#include "spirv.hpp" #include "spirv_cross_containers.hpp" #include "spirv_cross_error_handling.hpp" #include @@ -220,7 +229,7 @@ static inline std::string convert_to_string(int32_t value) // INT_MIN is ... special on some backends. If we use a decimal literal, and negate it, we // could accidentally promote the literal to long first, then negate. // To workaround it, emit int(0x80000000) instead. - if (value == std::numeric_limits::min()) + if (value == (std::numeric_limits::min)()) return "int(0x80000000)"; else return std::to_string(value); @@ -231,7 +240,7 @@ static inline std::string convert_to_string(int64_t value, const std::string &in // INT64_MIN is ... special on some backends. // If we use a decimal literal, and negate it, we might overflow the representable numbers. // To workaround it, emit int(0x80000000) instead. - if (value == std::numeric_limits::min()) + if (value == (std::numeric_limits::min)()) return join(int64_type, "(0x8000000000000000u", (long_long_literal_suffix ? "ll" : "l"), ")"); else return std::to_string(value) + (long_long_literal_suffix ? "ll" : "l"); @@ -368,6 +377,7 @@ enum Types TypeAccessChain, TypeUndef, TypeString, + TypeDebugLocalVariable, TypeCount }; @@ -497,6 +507,18 @@ struct SPIRString : IVariant SPIRV_CROSS_DECLARE_CLONE(SPIRString) }; +struct SPIRDebugLocalVariable : IVariant +{ + enum + { + type = TypeDebugLocalVariable + }; + + uint32_t name_id; + + SPIRV_CROSS_DECLARE_CLONE(SPIRDebugLocalVariable) +}; + // This type is only used by backends which need to access the combined image and sampler IDs separately after // the OpSampledImage opcode. struct SPIRCombinedImageSampler : IVariant @@ -574,11 +596,20 @@ struct SPIRType : IVariant Sampler, AccelerationStructure, RayQuery, + CoopVecNV, // Keep internal types at the end. ControlPointArray, Interpolant, - Char + Char, + // MSL specific type, that is used by 'object'(analog of 'task' from glsl) shader. + MeshGridProperties, + BFloat16, + FloatE4M3, + FloatE5M2, + + Tensor, + DescriptorHeapBuffer }; // Scalar/vector/matrix support. @@ -603,6 +634,35 @@ struct SPIRType : IVariant bool pointer = false; bool forward_pointer = false; + union + { + struct + { + uint32_t use_id; + uint32_t rows_id; + uint32_t columns_id; + uint32_t scope_id; + } cooperative; + + struct + { + uint32_t component_type_id; + uint32_t component_count_id; + } coopVecNV; + + struct + { + uint32_t type; + uint32_t rank; + uint32_t shape; + } tensor; + + struct + { + spv::StorageClass storage; + } descriptor_heap_buffer; + } ext; + spv::StorageClass storage = spv::StorageClassGeneric; SmallVector member_types; @@ -659,6 +719,12 @@ struct SPIRExtension : IVariant NonSemanticGeneric }; + enum ShaderDebugInfoOps + { + DebugLine = 103, + DebugSource = 35 + }; + explicit SPIRExtension(Extension ext_) : ext(ext_) { @@ -684,6 +750,11 @@ struct SPIREntryPoint FunctionID self = 0; std::string name; std::string orig_name; + std::unordered_map fp_fast_math_defaults; + bool signed_zero_inf_nan_preserve_8 = false; + bool signed_zero_inf_nan_preserve_16 = false; + bool signed_zero_inf_nan_preserve_32 = false; + bool signed_zero_inf_nan_preserve_64 = false; SmallVector interface_variables; Bitset flags; @@ -743,9 +814,20 @@ struct SPIRExpression : IVariant // Whether or not gl_MeshVerticesEXT[].gl_Position (as a whole or .y) is referenced bool access_meshlet_position_y = false; + // If this expression represents a OpBufferPointerEXT cast. + bool buffer_pointer = false; + + // Temporaries which can remain forwarded as long as this variable is not modified. + // Only used for buffer pointers. + SmallVector buffer_pointer_dependees; + // A list of expressions which this expression depends on. SmallVector expression_dependencies; + // Similar as expression dependencies, but does not stop the tracking for force-temporary variables. + // We need to know the full chain from store back to any SSA variable. + SmallVector invariance_dependencies; + // By reading this expression, we implicitly read these expressions as well. // Used by access chain Store and Load since we read multiple expressions in this case. SmallVector implied_read_expressions; @@ -912,6 +994,7 @@ struct SPIRBlock : IVariant // All access to these variables are dominated by this block, // so before branching anywhere we need to make sure that we declare these variables. SmallVector dominated_variables; + SmallVector rearm_dominated_variables; // These are variables which should be declared in a for loop header, if we // fail to use a classic for-loop, @@ -1020,6 +1103,9 @@ struct SPIRFunction : IVariant // consider arrays value types. SmallVector constant_arrays_needed_on_stack; + // Does this function (or any function called by it), emit geometry? + bool emits_geometry = false; + bool active = false; bool flush_undeclared = true; bool do_combined_parameters = true; @@ -1103,6 +1189,9 @@ struct SPIRVariable : IVariant // Temporaries which can remain forwarded as long as this variable is not modified. SmallVector dependees; + // ShaderDebugInfo local variables attached to this variable via DebugDeclare + SmallVector debug_local_variables; + bool deferred_declaration = false; bool phi_variable = false; @@ -1121,6 +1210,15 @@ struct SPIRVariable : IVariant // Set to true while we're inside the for loop. bool loop_variable_enable = false; + // Used to find global LUTs + bool is_written_to = false; + + // Untyped pointer. The pointer of the variable is effectively void. + // The underlying payload for allocation is in alloca_type, but may be 0 too. + // This is mostly here to support descriptor heap proxy. + bool untyped = false; + ID untyped_alloca_type = 0; + SPIRFunction::Parameter *parameter = nullptr; SPIRV_CROSS_DECLARE_CLONE(SPIRVariable) @@ -1217,6 +1315,26 @@ struct SPIRConstant : IVariant return u.f32; } + static inline float fe4m3_to_f32(uint8_t v) + { + if ((v & 0x7f) == 0x7f) + { + union + { + float f32; + uint32_t u32; + } u; + + u.u32 = (v & 0x80) ? 0xffffffffu : 0x7fffffffu; + return u.f32; + } + else + { + // Reuse the FP16 to FP32 code. Cute bit-hackery. + return f16_to_f32((int16_t(int8_t(v)) << 7) & (0xffff ^ 0x4000)) * 256.0f; + } + } + inline uint32_t specialization_constant_id(uint32_t col, uint32_t row) const { return m.c[col].id[row]; @@ -1257,6 +1375,24 @@ struct SPIRConstant : IVariant return f16_to_f32(scalar_u16(col, row)); } + inline float scalar_bf16(uint32_t col = 0, uint32_t row = 0) const + { + uint32_t v = scalar_u16(col, row) << 16; + float fp32; + memcpy(&fp32, &v, sizeof(float)); + return fp32; + } + + inline float scalar_floate4m3(uint32_t col = 0, uint32_t row = 0) const + { + return fe4m3_to_f32(scalar_u8(col, row)); + } + + inline float scalar_bf8(uint32_t col = 0, uint32_t row = 0) const + { + return f16_to_f32(uint16_t(scalar_u8(col, row) << 8)); + } + inline float scalar_f32(uint32_t col = 0, uint32_t row = 0) const { return m.c[col].r[row].f32; @@ -1327,9 +1463,10 @@ struct SPIRConstant : IVariant SPIRConstant() = default; - SPIRConstant(TypeID constant_type_, const uint32_t *elements, uint32_t num_elements, bool specialized) + SPIRConstant(TypeID constant_type_, const uint32_t *elements, uint32_t num_elements, bool specialized, bool replicated_ = false) : constant_type(constant_type_) , specialization(specialized) + , replicated(replicated_) { subconstants.reserve(num_elements); for (uint32_t i = 0; i < num_elements; i++) @@ -1401,15 +1538,25 @@ struct SPIRConstant : IVariant // If true, this is a LUT, and should always be declared in the outer scope. bool is_used_as_lut = false; + // If this is a null constant of array type with specialized length. + // May require special handling in initializer + bool is_null_array_specialized_length = false; + // For composites which are constant arrays, etc. SmallVector subconstants; + // Whether the subconstants are intended to be replicated (e.g. OpConstantCompositeReplicateEXT) + bool replicated = false; + // Non-Vulkan GLSL, HLSL and sometimes MSL emits defines for each specialization constant, // and uses them to initialize the constant. This allows the user // to still be able to specialize the value by supplying corresponding // preprocessor directives before compiling the shader. std::string specialization_constant_macro_name; + // ConstantSizeOfEXT. + ID size_of_type = 0; + SPIRV_CROSS_DECLARE_CLONE(SPIRConstant) }; @@ -1595,6 +1742,8 @@ struct AccessChainMeta bool flattened_struct = false; bool relaxed_precision = false; bool access_meshlet_position_y = false; + bool chain_is_builtin = false; + spv::BuiltIn builtin = {}; }; enum ExtendedDecorations @@ -1668,6 +1817,8 @@ enum ExtendedDecorations // lack of constructors in the 'threadgroup' address space. SPIRVCrossDecorationWorkgroupStruct, + SPIRVCrossDecorationOverlappingBinding, + SPIRVCrossDecorationCount }; @@ -1677,7 +1828,7 @@ struct Meta { std::string alias; std::string qualified_alias; - std::string hlsl_semantic; + std::string user_semantic; std::string user_type; Bitset decoration_flags; spv::BuiltIn builtin_type = spv::BuiltInMax; @@ -1686,16 +1837,20 @@ struct Meta uint32_t set = 0; uint32_t binding = 0; uint32_t offset = 0; + uint32_t offset_id = 0; uint32_t xfb_buffer = 0; uint32_t xfb_stride = 0; uint32_t stream = 0; uint32_t array_stride = 0; + uint32_t array_stride_id = 0; uint32_t matrix_stride = 0; uint32_t input_attachment = 0; uint32_t spec_id = 0; uint32_t index = 0; spv::FPRoundingMode fp_rounding_mode = spv::FPRoundingModeMax; + spv::FPFastMathModeMask fp_fast_math_mode = spv::FPFastMathModeMaskNone; bool builtin = false; + bool qualified_alias_explicit_override = false; struct Extended { @@ -1749,7 +1904,8 @@ private: static inline bool type_is_floating_point(const SPIRType &type) { - return type.basetype == SPIRType::Half || type.basetype == SPIRType::Float || type.basetype == SPIRType::Double; + return type.basetype == SPIRType::Half || type.basetype == SPIRType::Float || type.basetype == SPIRType::Double || + type.basetype == SPIRType::BFloat16 || type.basetype == SPIRType::FloatE5M2 || type.basetype == SPIRType::FloatE4M3; } static inline bool type_is_integral(const SPIRType &type) @@ -1934,4 +2090,7 @@ struct hash> }; } // namespace std +#ifdef SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#undef spv +#endif #endif diff --git a/src/libraries/spirv_cross/spirv_cpp.cpp b/src/libraries/spirv_cross/spirv_cpp.cpp index dd0a84c83..4bb6fc261 100644 --- a/src/libraries/spirv_cross/spirv_cpp.cpp +++ b/src/libraries/spirv_cross/spirv_cpp.cpp @@ -23,7 +23,7 @@ #include "spirv_cpp.hpp" -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; using namespace std; @@ -40,7 +40,7 @@ void CompilerCPP::emit_buffer_block(const SPIRVariable &var) emit_block_struct(type); auto buffer_name = to_name(type.self); - statement("internal::Resource<", buffer_name, type_to_array_glsl(type), "> ", instance_name, "__;"); + statement("internal::Resource<", buffer_name, type_to_array_glsl(type, var.self), "> ", instance_name, "__;"); statement_no_indent("#define ", instance_name, " __res->", instance_name, "__.get()"); resource_registrations.push_back( join("s.register_resource(", instance_name, "__", ", ", descriptor_set, ", ", binding, ");")); @@ -68,7 +68,7 @@ void CompilerCPP::emit_interface_block(const SPIRVariable &var) else buffer_name = type_to_glsl(type); - statement("internal::", qual, "<", buffer_name, type_to_array_glsl(type), "> ", instance_name, "__;"); + statement("internal::", qual, "<", buffer_name, type_to_array_glsl(type, var.self), "> ", instance_name, "__;"); statement_no_indent("#define ", instance_name, " __res->", instance_name, "__.get()"); resource_registrations.push_back(join("s.register_", lowerqual, "(", instance_name, "__", ", ", location, ");")); statement(""); @@ -100,14 +100,14 @@ void CompilerCPP::emit_uniform(const SPIRVariable &var) if (type.basetype == SPIRType::Image || type.basetype == SPIRType::SampledImage || type.basetype == SPIRType::AtomicCounter) { - statement("internal::Resource<", type_name, type_to_array_glsl(type), "> ", instance_name, "__;"); + statement("internal::Resource<", type_name, type_to_array_glsl(type, var.self), "> ", instance_name, "__;"); statement_no_indent("#define ", instance_name, " __res->", instance_name, "__.get()"); resource_registrations.push_back( join("s.register_resource(", instance_name, "__", ", ", descriptor_set, ", ", binding, ");")); } else { - statement("internal::UniformConstant<", type_name, type_to_array_glsl(type), "> ", instance_name, "__;"); + statement("internal::UniformConstant<", type_name, type_to_array_glsl(type, var.self), "> ", instance_name, "__;"); statement_no_indent("#define ", instance_name, " __res->", instance_name, "__.get()"); resource_registrations.push_back( join("s.register_uniform_constant(", instance_name, "__", ", ", location, ");")); @@ -130,7 +130,7 @@ void CompilerCPP::emit_push_constant_block(const SPIRVariable &var) auto buffer_name = to_name(type.self); auto instance_name = to_name(var.self); - statement("internal::PushConstant<", buffer_name, type_to_array_glsl(type), "> ", instance_name, ";"); + statement("internal::PushConstant<", buffer_name, type_to_array_glsl(type, var.self), "> ", instance_name, ";"); statement_no_indent("#define ", instance_name, " __res->", instance_name, ".get()"); resource_registrations.push_back(join("s.register_push_constant(", instance_name, "__", ");")); statement(""); diff --git a/src/libraries/spirv_cross/spirv_cross.cpp b/src/libraries/spirv_cross/spirv_cross.cpp index 4ab985efd..810084e96 100644 --- a/src/libraries/spirv_cross/spirv_cross.cpp +++ b/src/libraries/spirv_cross/spirv_cross.cpp @@ -31,7 +31,7 @@ #include using namespace std; -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; Compiler::Compiler(vector ir_) @@ -78,11 +78,16 @@ string Compiler::compile() bool Compiler::variable_storage_is_aliased(const SPIRVariable &v) { auto &type = get(v.basetype); + + // Untyped pointer, assume full aliasing. + if (type.basetype == SPIRType::Void) + return true; + bool ssbo = v.storage == StorageClassStorageBuffer || ir.meta[type.self].decoration.decoration_flags.get(DecorationBufferBlock); bool image = type.basetype == SPIRType::Image; bool counter = type.basetype == SPIRType::AtomicCounter; - bool buffer_reference = type.storage == StorageClassPhysicalStorageBufferEXT; + bool buffer_reference = type.storage == StorageClassPhysicalStorageBuffer; bool is_restrict; if (ssbo) @@ -93,6 +98,98 @@ bool Compiler::variable_storage_is_aliased(const SPIRVariable &v) return !is_restrict && (ssbo || image || counter || buffer_reference); } +bool Compiler::block_is_control_dependent(const SPIRBlock &block) +{ + for (auto &i : block.ops) + { + auto ops = stream(i); + auto op = static_cast(i.op); + + switch (op) + { + case OpFunctionCall: + { + uint32_t func = ops[2]; + if (function_is_control_dependent(get(func))) + return true; + break; + } + + // Derivatives + case OpDPdx: + case OpDPdxCoarse: + case OpDPdxFine: + case OpDPdy: + case OpDPdyCoarse: + case OpDPdyFine: + case OpFwidth: + case OpFwidthCoarse: + case OpFwidthFine: + + // Anything implicit LOD + case OpImageSampleImplicitLod: + case OpImageSampleDrefImplicitLod: + case OpImageSampleProjImplicitLod: + case OpImageSampleProjDrefImplicitLod: + case OpImageSparseSampleImplicitLod: + case OpImageSparseSampleDrefImplicitLod: + case OpImageSparseSampleProjImplicitLod: + case OpImageSparseSampleProjDrefImplicitLod: + case OpImageQueryLod: + case OpImageDrefGather: + case OpImageGather: + case OpImageSparseDrefGather: + case OpImageSparseGather: + + // Anything subgroups + case OpGroupNonUniformElect: + case OpGroupNonUniformAll: + case OpGroupNonUniformAny: + case OpGroupNonUniformAllEqual: + case OpGroupNonUniformBroadcast: + case OpGroupNonUniformBroadcastFirst: + case OpGroupNonUniformBallot: + case OpGroupNonUniformInverseBallot: + case OpGroupNonUniformBallotBitExtract: + case OpGroupNonUniformBallotBitCount: + case OpGroupNonUniformBallotFindLSB: + case OpGroupNonUniformBallotFindMSB: + case OpGroupNonUniformShuffle: + case OpGroupNonUniformShuffleXor: + case OpGroupNonUniformShuffleUp: + case OpGroupNonUniformShuffleDown: + case OpGroupNonUniformIAdd: + case OpGroupNonUniformFAdd: + case OpGroupNonUniformIMul: + case OpGroupNonUniformFMul: + case OpGroupNonUniformSMin: + case OpGroupNonUniformUMin: + case OpGroupNonUniformFMin: + case OpGroupNonUniformSMax: + case OpGroupNonUniformUMax: + case OpGroupNonUniformFMax: + case OpGroupNonUniformBitwiseAnd: + case OpGroupNonUniformBitwiseOr: + case OpGroupNonUniformBitwiseXor: + case OpGroupNonUniformLogicalAnd: + case OpGroupNonUniformLogicalOr: + case OpGroupNonUniformLogicalXor: + case OpGroupNonUniformQuadBroadcast: + case OpGroupNonUniformQuadSwap: + case OpGroupNonUniformRotateKHR: + + // Control barriers + case OpControlBarrier: + return true; + + default: + break; + } + } + + return false; +} + bool Compiler::block_is_pure(const SPIRBlock &block) { // This is a global side effect of the function. @@ -119,6 +216,7 @@ bool Compiler::block_is_pure(const SPIRBlock &block) case OpCopyMemory: case OpStore: + case OpCooperativeMatrixStoreKHR: { auto &type = expression_type(ops[0]); if (type.storage != StorageClassFunction) @@ -187,6 +285,9 @@ bool Compiler::block_is_pure(const SPIRBlock &block) // This is a global side effect of the function. return false; + case OpTensorReadARM: + return false; + case OpExtInst: { uint32_t extension_set = ops[2]; @@ -247,18 +348,21 @@ string Compiler::to_name(uint32_t id, bool allow_alias) const bool Compiler::function_is_pure(const SPIRFunction &func) { for (auto block : func.blocks) - { if (!block_is_pure(get(block))) - { - //fprintf(stderr, "Function %s is impure!\n", to_name(func.self).c_str()); return false; - } - } - //fprintf(stderr, "Function %s is pure!\n", to_name(func.self).c_str()); return true; } +bool Compiler::function_is_control_dependent(const SPIRFunction &func) +{ + for (auto block : func.blocks) + if (block_is_control_dependent(get(block))) + return true; + + return false; +} + void Compiler::register_global_read_dependencies(const SPIRBlock &block, uint32_t id) { for (auto &i : block.ops) @@ -276,6 +380,8 @@ void Compiler::register_global_read_dependencies(const SPIRBlock &block, uint32_ } case OpLoad: + case OpCooperativeMatrixLoadKHR: + case OpCooperativeVectorLoadNV: case OpImageRead: { // If we're in a storage class which does not get invalidated, adding dependencies here is no big deal. @@ -310,20 +416,37 @@ SPIRVariable *Compiler::maybe_get_backing_variable(uint32_t chain) { auto *cexpr = maybe_get(chain); if (cexpr) + { var = maybe_get(cexpr->loaded_from); + if (!var && cexpr->loaded_from != chain) + var = maybe_get_backing_variable(cexpr->loaded_from); + } auto *access_chain = maybe_get(chain); if (access_chain) + { var = maybe_get(access_chain->loaded_from); + if (!var && access_chain->loaded_from != chain) + var = maybe_get_backing_variable(access_chain->loaded_from); + } } return var; } +SPIRExpression *Compiler::maybe_get_backing_buffer_pointer(uint32_t chain) +{ + auto *expr = maybe_get(chain); + while (expr && !expr->buffer_pointer && expr->loaded_from) + expr = maybe_get(expr->loaded_from); + return expr && expr->buffer_pointer ? expr : nullptr; +} + void Compiler::register_read(uint32_t expr, uint32_t chain, bool forwarded) { auto &e = get(expr); auto *var = maybe_get_backing_variable(chain); + auto *buffer_pointer = maybe_get_backing_buffer_pointer(chain); if (var) { @@ -338,6 +461,13 @@ void Compiler::register_read(uint32_t expr, uint32_t chain, bool forwarded) if (var && var->parameter) var->parameter->read_count++; } + else if (buffer_pointer) + { + e.loaded_from = buffer_pointer->self; + // If the backing variable is immutable, we do not need to depend on the variable. + if (forwarded && !is_immutable(buffer_pointer->self)) + buffer_pointer->buffer_pointer_dependees.push_back(e.self); + } } void Compiler::register_write(uint32_t chain) @@ -355,6 +485,8 @@ void Compiler::register_write(uint32_t chain) var = maybe_get(access_chain->loaded_from); } + auto *buffer_pointer = maybe_get_backing_buffer_pointer(chain); + auto &chain_type = expression_type(chain); if (var) @@ -387,7 +519,7 @@ void Compiler::register_write(uint32_t chain) } } - if (type.storage == StorageClassPhysicalStorageBufferEXT || variable_storage_is_aliased(*var)) + if (type.storage == StorageClassPhysicalStorageBuffer || variable_storage_is_aliased(*var)) flush_all_aliased_variables(); else if (var) flush_dependees(*var); @@ -399,6 +531,10 @@ void Compiler::register_write(uint32_t chain) force_recompile(); } } + else if (buffer_pointer) + { + flush_dependees(*buffer_pointer); + } else if (chain_type.pointer) { // If we stored through a variable pointer, then we don't know which @@ -420,6 +556,16 @@ void Compiler::flush_dependees(SPIRVariable &var) var.dependees.clear(); } +void Compiler::flush_dependees(SPIRExpression &expr) +{ + // A little ugly to split things up like this since BufferPointerEXT is a weird case + // where it's both an expression (chain into global heap) and a memory declaration at the same time ... + assert(expr.buffer_pointer); + for (auto dep : expr.buffer_pointer_dependees) + invalid_expressions.insert(dep); + expr.buffer_pointer_dependees.clear(); +} + void Compiler::flush_all_aliased_variables() { for (auto aliased : aliased_variables) @@ -430,6 +576,8 @@ void Compiler::flush_all_atomic_capable_variables() { for (auto global : global_variables) flush_dependees(get(global)); + for (auto global : buffer_pointer_variables) + flush_dependees(get(global)); flush_all_aliased_variables(); } @@ -451,6 +599,8 @@ void Compiler::flush_all_active_variables() flush_dependees(get(arg.id)); for (auto global : global_variables) flush_dependees(get(global)); + for (auto global : buffer_pointer_variables) + flush_dependees(get(global)); flush_all_aliased_variables(); } @@ -493,6 +643,7 @@ const SPIRType &Compiler::expression_type(uint32_t id) const bool Compiler::expression_is_lvalue(uint32_t id) const { auto &type = expression_type(id); + switch (type.basetype) { case SPIRType::SampledImage: @@ -526,7 +677,7 @@ bool Compiler::is_immutable(uint32_t id) const return false; } -static inline bool storage_class_is_interface(spv::StorageClass storage) +static inline bool storage_class_is_interface(StorageClass storage) { switch (storage) { @@ -558,9 +709,10 @@ bool Compiler::is_hidden_variable(const SPIRVariable &var, bool include_builtins } // In SPIR-V 1.4 and up we must also use the active variable interface to disable global variables - // which are not part of the entry point. - if (ir.get_spirv_version() >= 0x10400 && var.storage != spv::StorageClassGeneric && - var.storage != spv::StorageClassFunction && !interface_variable_exists_in_entry_point(var.self)) + // which are not part of the entry point. Library modules have no real entry point so the filter + // would hide every global so skip it in that case. + if (ir.get_spirv_version() >= 0x10400 && !ir.is_library_module && var.storage != StorageClassGeneric && + var.storage != StorageClassFunction && !interface_variable_exists_in_entry_point(var.self)) { return true; } @@ -632,12 +784,27 @@ bool Compiler::is_array(const SPIRType &type) const bool Compiler::is_pointer(const SPIRType &type) const { - return type.op == OpTypePointer && type.basetype != SPIRType::Unknown; // Ignore function pointers. + return (type.op == OpTypePointer || type.op == OpTypeUntypedPointerKHR) && type.basetype != SPIRType::Unknown; // Ignore function pointers. } bool Compiler::is_physical_pointer(const SPIRType &type) const { - return type.op == OpTypePointer && type.storage == StorageClassPhysicalStorageBuffer; + return (type.op == OpTypePointer || type.op == OpTypeUntypedPointerKHR) && type.storage == StorageClassPhysicalStorageBuffer; +} + +bool Compiler::is_physical_or_buffer_pointer(const SPIRType &type) const +{ + return (type.op == OpTypePointer || type.op == OpTypeUntypedPointerKHR) && + (type.storage == StorageClassPhysicalStorageBuffer || type.storage == StorageClassUniform || + type.storage == StorageClassStorageBuffer || type.storage == StorageClassWorkgroup || + type.storage == StorageClassPushConstant); +} + +bool Compiler::is_physical_pointer_to_buffer_block(const SPIRType &type) const +{ + return is_physical_pointer(type) && get_pointee_type(type).self == type.parent_type && + (has_decoration(type.self, DecorationBlock) || + has_decoration(type.self, DecorationBufferBlock)); } bool Compiler::is_runtime_size_array(const SPIRType &type) @@ -717,6 +884,7 @@ bool Compiler::InterfaceVariableAccessHandler::handle(Op opcode, const uint32_t case OpAtomicStore: case OpStore: + case OpCooperativeMatrixStoreKHR: // Invalid SPIR-V. if (length < 1) return false; @@ -809,6 +977,7 @@ bool Compiler::InterfaceVariableAccessHandler::handle(Op opcode, const uint32_t case OpInBoundsAccessChain: case OpPtrAccessChain: case OpLoad: + case OpCooperativeMatrixLoadKHR: case OpCopyObject: case OpImageTexelPointer: case OpAtomicLoad: @@ -1045,6 +1214,11 @@ ShaderResources Compiler::get_shader_resources(const unordered_set * { res.acceleration_structures.push_back({ var.self, var.basetype, type.self, get_name(var.self) }); } + // Tensors + else if (type.basetype == SPIRType::Tensor) + { + res.tensors.push_back({ var.self, var.basetype, type.self, get_name(var.self) }); + } else { res.gl_plain_uniforms.push_back({ var.self, var.basetype, type.self, get_name(var.self) }); @@ -1062,11 +1236,8 @@ bool Compiler::type_is_top_level_block(const SPIRType &type) const return has_decoration(type.self, DecorationBlock) || has_decoration(type.self, DecorationBufferBlock); } -bool Compiler::type_is_block_like(const SPIRType &type) const +bool Compiler::type_is_explicit_layout(const SPIRType &type) const { - if (type_is_top_level_block(type)) - return true; - if (type.basetype == SPIRType::Struct) { // Block-like types may have Offset decorations. @@ -1078,6 +1249,14 @@ bool Compiler::type_is_block_like(const SPIRType &type) const return false; } +bool Compiler::type_is_block_like(const SPIRType &type) const +{ + if (type_is_top_level_block(type)) + return true; + else + return type_is_explicit_layout(type); +} + void Compiler::parse_fixup() { // Figure out specialization constants for work group sizes. @@ -1220,7 +1399,7 @@ const SPIRType &Compiler::get_pointee_type(uint32_t type_id) const uint32_t Compiler::get_variable_data_type_id(const SPIRVariable &var) const { - if (var.phi_variable) + if (var.phi_variable || var.storage == StorageClassAtomicCounter) return var.basetype; return get_pointee_type_id(var.basetype); } @@ -1257,7 +1436,7 @@ bool Compiler::is_sampled_image_type(const SPIRType &type) type.image.dim != DimBuffer; } -void Compiler::set_member_decoration_string(TypeID id, uint32_t index, spv::Decoration decoration, +void Compiler::set_member_decoration_string(TypeID id, uint32_t index, Decoration decoration, const std::string &argument) { ir.set_member_decoration_string(id, index, decoration, argument); @@ -1318,7 +1497,7 @@ void Compiler::unset_member_decoration(TypeID id, uint32_t index, Decoration dec ir.unset_member_decoration(id, index, decoration); } -void Compiler::set_decoration_string(ID id, spv::Decoration decoration, const std::string &argument) +void Compiler::set_decoration_string(ID id, Decoration decoration, const std::string &argument) { ir.set_decoration_string(id, decoration, argument); } @@ -1481,7 +1660,7 @@ void Compiler::unset_decoration(ID id, Decoration decoration) ir.unset_decoration(id, decoration); } -bool Compiler::get_binary_offset_for_decoration(VariableID id, spv::Decoration decoration, uint32_t &word_offset) const +bool Compiler::get_binary_offset_for_decoration(VariableID id, Decoration decoration, uint32_t &word_offset) const { auto *m = ir.find_meta(id); if (!m) @@ -1749,6 +1928,11 @@ const SmallVector &Compiler::get_case_list(const SPIRBlock &blo const auto &type = get(constant->constant_type); width = type.width; } + else if (const auto *op = maybe_get(block.condition)) + { + const auto &type = get(op->basetype); + width = type.width; + } else if (const auto *var = maybe_get(block.condition)) { const auto &type = get(var->basetype); @@ -1781,6 +1965,15 @@ bool Compiler::traverse_all_reachable_opcodes(const SPIRBlock &block, OpcodeHand handler.set_current_block(block); handler.rearm_current_block(block); + if (handler.enable_result_types) + { + for (auto &phi: block.phi_variables) + { + auto &v = get(phi.function_variable); + handler.result_types[phi.function_variable] = v.basetype; + } + } + // Ideally, perhaps traverse the CFG instead of all blocks in order to eliminate dead blocks, // but this shouldn't be a problem in practice unless the SPIR-V is doing insane things like recursing // inside dead blocks ... @@ -1792,11 +1985,24 @@ bool Compiler::traverse_all_reachable_opcodes(const SPIRBlock &block, OpcodeHand if (!handler.handle(op, ops, i.length)) return false; + if (handler.enable_result_types) + { + // If it has one, keep track of the instruction's result type, mapped by ID + uint32_t result_type, result_id; + if (instruction_to_result_type(result_type, result_id, op, ops, i.length)) + handler.result_types[result_id] = result_type; + } + if (op == OpFunctionCall) { auto &func = get(ops[2]); if (handler.follow_function_call(func)) { + if (handler.enable_result_types) + for (auto &arg : func.arguments) + if (!arg.alias_global_variable) + handler.result_types[arg.id] = arg.type; + if (!handler.begin_function_scope(ops, i.length)) return false; if (!traverse_all_reachable_opcodes(get(ops[2]), handler)) @@ -1904,7 +2110,7 @@ size_t Compiler::get_declared_struct_size_runtime_array(const SPIRType &type, si size_t size = get_declared_struct_size(type); auto &last_type = get(type.member_types.back()); - if (!last_type.array.empty() && last_type.array_size_literal[0] && last_type.array[0] == 0) // Runtime array + if (!last_type.array.empty() && last_type.array_size_literal.back() && last_type.array.back() == 0) // Runtime array size += array_size * type_struct_member_array_stride(type, uint32_t(type.member_types.size() - 1)); return size; @@ -2258,6 +2464,10 @@ void Compiler::set_execution_mode(ExecutionMode mode, uint32_t arg0, uint32_t ar execution.output_primitives = arg0; break; + case ExecutionModeFPFastMathDefault: + execution.fp_fast_math_defaults[arg0] = arg1; + break; + default: break; } @@ -2327,7 +2537,7 @@ uint32_t Compiler::get_work_group_size_specialization_constants(SpecializationCo return execution.workgroup_size.constant; } -uint32_t Compiler::get_execution_mode_argument(spv::ExecutionMode mode, uint32_t index) const +uint32_t Compiler::get_execution_mode_argument(ExecutionMode mode, uint32_t index) const { auto &execution = get_entry_point(); switch (mode) @@ -2463,6 +2673,15 @@ void Compiler::add_active_interface_variable(uint32_t var_id) void Compiler::inherit_expression_dependencies(uint32_t dst, uint32_t source_expression) { + auto *ptr_e = maybe_get(dst); + + if (is_position_invariant() && ptr_e && maybe_get(source_expression)) + { + auto &deps = ptr_e->invariance_dependencies; + if (std::find(deps.begin(), deps.end(), source_expression) == deps.end()) + deps.push_back(source_expression); + } + // Don't inherit any expression dependencies if the expression in dst // is not a forwarded temporary. if (forwarded_temporaries.find(dst) == end(forwarded_temporaries) || @@ -2471,7 +2690,7 @@ void Compiler::inherit_expression_dependencies(uint32_t dst, uint32_t source_exp return; } - auto &e = get(dst); + auto &e = *ptr_e; auto *phi = maybe_get(source_expression); if (phi && phi->phi_variable) { @@ -2504,14 +2723,14 @@ SmallVector Compiler::get_entry_points_and_stages() const return entries; } -void Compiler::rename_entry_point(const std::string &old_name, const std::string &new_name, spv::ExecutionModel model) +void Compiler::rename_entry_point(const std::string &old_name, const std::string &new_name, ExecutionModel model) { auto &entry = get_entry_point(old_name, model); entry.orig_name = new_name; entry.name = new_name; } -void Compiler::set_entry_point(const std::string &name, spv::ExecutionModel model) +void Compiler::set_entry_point(const std::string &name, ExecutionModel model) { auto &entry = get_entry_point(name, model); ir.default_entry_point = entry.self; @@ -3207,7 +3426,7 @@ void Compiler::analyze_parameter_preservation( Compiler::AnalyzeVariableScopeAccessHandler::AnalyzeVariableScopeAccessHandler(Compiler &compiler_, SPIRFunction &entry_) - : compiler(compiler_) + : OpcodeHandler(compiler_) , entry(entry_) { } @@ -3325,16 +3544,14 @@ bool Compiler::AnalyzeVariableScopeAccessHandler::handle_terminator(const SPIRBl return true; } -bool Compiler::AnalyzeVariableScopeAccessHandler::handle(spv::Op op, const uint32_t *args, uint32_t length) +bool Compiler::AnalyzeVariableScopeAccessHandler::handle(Op op, const uint32_t *args, uint32_t length) { // Keep track of the types of temporaries, so we can hoist them out as necessary. - uint32_t result_type, result_id; - if (compiler.instruction_to_result_type(result_type, result_id, op, args, length)) + uint32_t result_type = 0, result_id = 0; + if (instruction_to_result_type(result_type, result_id, op, args, length)) { // For some opcodes, we will need to override the result id. // If we need to hoist the temporary, the temporary type is the input, not the result. - // FIXME: This will likely break with OpCopyObject + hoisting, but we'll have to - // solve it if we ever get there ... if (op == OpConvertUToAccelerationStructureKHR) { auto itr = result_id_to_type.find(args[2]); @@ -3348,6 +3565,7 @@ bool Compiler::AnalyzeVariableScopeAccessHandler::handle(spv::Op op, const uint3 switch (op) { case OpStore: + case OpCooperativeMatrixStoreKHR: { if (length < 2) return false; @@ -3443,6 +3661,13 @@ bool Compiler::AnalyzeVariableScopeAccessHandler::handle(spv::Op op, const uint3 case OpCopyObject: { + // OpCopyObject copies the underlying non-pointer type, + // so any temp variable should be declared using the underlying type. + // If the type is a pointer, get its base type and overwrite the result type mapping. + auto &type = compiler.get(result_type); + if (type.pointer) + result_id_to_type[result_id] = type.parent_type; + if (length < 3) return false; @@ -3461,6 +3686,7 @@ bool Compiler::AnalyzeVariableScopeAccessHandler::handle(spv::Op op, const uint3 } case OpLoad: + case OpCooperativeMatrixLoadKHR: { if (length < 3) return false; @@ -3665,7 +3891,7 @@ bool Compiler::AnalyzeVariableScopeAccessHandler::handle(spv::Op op, const uint3 } Compiler::StaticExpressionAccessHandler::StaticExpressionAccessHandler(Compiler &compiler_, uint32_t variable_id_) - : compiler(compiler_) + : OpcodeHandler(compiler_) , variable_id(variable_id_) { } @@ -3675,11 +3901,12 @@ bool Compiler::StaticExpressionAccessHandler::follow_function_call(const SPIRFun return false; } -bool Compiler::StaticExpressionAccessHandler::handle(spv::Op op, const uint32_t *args, uint32_t length) +bool Compiler::StaticExpressionAccessHandler::handle(Op op, const uint32_t *args, uint32_t length) { switch (op) { case OpStore: + case OpCooperativeMatrixStoreKHR: if (length < 2) return false; if (args[0] == variable_id) @@ -3690,6 +3917,7 @@ bool Compiler::StaticExpressionAccessHandler::handle(spv::Op op, const uint32_t break; case OpLoad: + case OpCooperativeMatrixLoadKHR: if (length < 3) return false; if (args[2] == variable_id && static_expression == 0) // Tried to read from variable before it was initialized. @@ -3724,6 +3952,14 @@ void Compiler::find_function_local_luts(SPIRFunction &entry, const AnalyzeVariab auto &var = get(accessed_var.first); auto &type = expression_type(accessed_var.first); + // First check if there are writes to the variable. Later, if there are none, we'll + // reconsider it as globally accessed LUT. + if (!var.is_written_to) + { + var.is_written_to = handler.complete_write_variables_to_block.count(var.self) != 0 || + handler.partial_write_variables_to_block.count(var.self) != 0; + } + // Only consider function local variables here. // If we only have a single function in our CFG, private storage is also fine, // since it behaves like a function local variable. @@ -3748,8 +3984,7 @@ void Compiler::find_function_local_luts(SPIRFunction &entry, const AnalyzeVariab static_constant_expression = var.initializer; // There can be no stores to this variable, we have now proved we have a LUT. - if (handler.complete_write_variables_to_block.count(var.self) != 0 || - handler.partial_write_variables_to_block.count(var.self) != 0) + if (var.is_written_to) continue; } else @@ -4158,6 +4393,7 @@ bool Compiler::may_read_undefined_variable_in_block(const SPIRBlock &block, uint switch (op.op) { case OpStore: + case OpCooperativeMatrixStoreKHR: case OpCopyMemory: if (ops[0] == var) return false; @@ -4196,6 +4432,8 @@ bool Compiler::may_read_undefined_variable_in_block(const SPIRBlock &block, uint case OpCopyObject: case OpLoad: + case OpCooperativeVectorLoadNV: + case OpCooperativeMatrixLoadKHR: if (ops[2] == var) return true; break; @@ -4223,6 +4461,40 @@ bool Compiler::may_read_undefined_variable_in_block(const SPIRBlock &block, uint return true; } +bool Compiler::GeometryEmitDisocveryHandler::handle(Op opcode, const uint32_t *, uint32_t) +{ + if (opcode == OpEmitVertex || opcode == OpEndPrimitive) + { + for (auto *func : function_stack) + func->emits_geometry = true; + } + + return true; +} + +bool Compiler::GeometryEmitDisocveryHandler::begin_function_scope(const uint32_t *stream, uint32_t) +{ + auto &callee = compiler.get(stream[2]); + function_stack.push_back(&callee); + return true; +} + +bool Compiler::GeometryEmitDisocveryHandler::end_function_scope(const uint32_t *stream, uint32_t) +{ + (void)stream; + assert(function_stack.back() == &compiler.get(stream[2])); + function_stack.pop_back(); + + return true; +} + +void Compiler::discover_geometry_emitters() +{ + GeometryEmitDisocveryHandler handler(*this); + + traverse_all_reachable_opcodes(get(ir.default_entry_point), handler); +} + Bitset Compiler::get_buffer_block_flags(VariableID id) const { return ir.get_buffer_block_flags(get(id)); @@ -4330,11 +4602,12 @@ void Compiler::ActiveBuiltinHandler::add_if_builtin_or_block(uint32_t id) add_if_builtin(id, true); } -bool Compiler::ActiveBuiltinHandler::handle(spv::Op opcode, const uint32_t *args, uint32_t length) +bool Compiler::ActiveBuiltinHandler::handle(Op opcode, const uint32_t *args, uint32_t length) { switch (opcode) { case OpStore: + case OpCooperativeMatrixStoreKHR: if (length < 1) return false; @@ -4351,6 +4624,7 @@ bool Compiler::ActiveBuiltinHandler::handle(spv::Op opcode, const uint32_t *args case OpCopyObject: case OpLoad: + case OpCooperativeMatrixLoadKHR: if (length < 3) return false; @@ -4416,11 +4690,9 @@ bool Compiler::ActiveBuiltinHandler::handle(spv::Op opcode, const uint32_t *args for (uint32_t i = 0; i < count; i++) { // Pointers + // PtrAccessChain functions more like a pointer offset. Type remains the same. if (opcode == OpPtrAccessChain && i == 0) - { - type = &compiler.get(type->parent_type); continue; - } // Arrays if (!type->array.empty()) @@ -4525,7 +4797,7 @@ void Compiler::analyze_image_and_sampler_usage() comparison_ids.insert(combined.combined_id); } -bool Compiler::CombinedImageSamplerDrefHandler::handle(spv::Op opcode, const uint32_t *args, uint32_t) +bool Compiler::CombinedImageSamplerDrefHandler::handle(Op opcode, const uint32_t *args, uint32_t) { // Mark all sampled images which are used with Dref. switch (opcode) @@ -4569,6 +4841,18 @@ void Compiler::build_function_control_flow_graphs_and_analyze() CFGBuilder handler(*this); handler.function_cfgs[ir.default_entry_point].reset(new CFG(*this, get(ir.default_entry_point))); traverse_all_reachable_opcodes(get(ir.default_entry_point), handler); + if (ir.is_library_module) + { + // In library mode, default_entry_point is just the first exported + // function. Build a CFG for every other exported function (and its + // callees) so per-function analyses below cover all of them. + for (auto export_id : ir.library_exported_functions) + { + auto &func = get(export_id); + if (handler.follow_function_call(func)) + traverse_all_reachable_opcodes(func, handler); + } + } function_cfgs = std::move(handler.function_cfgs); bool single_function = function_cfgs.size() <= 1; @@ -4608,14 +4892,37 @@ void Compiler::build_function_control_flow_graphs_and_analyze() } } } + + // Find LUTs which are not function local. Only consider this case if the CFG is multi-function, + // otherwise we treat Private as Function trivially. + // Needs to be analyzed from the outside since we have to block the LUT optimization if at least + // one function writes to it. + if (!single_function) + { + for (auto &id : global_variables) + { + auto &var = get(id); + auto &type = get_variable_data_type(var); + + if (is_array(type) && var.storage == StorageClassPrivate && + var.initializer && !var.is_written_to && + ir.ids[var.initializer].get_type() == TypeConstant) + { + get(var.initializer).is_used_as_lut = true; + var.static_expression = var.initializer; + var.statically_assigned = true; + var.remapped_variable = true; + } + } + } } Compiler::CFGBuilder::CFGBuilder(Compiler &compiler_) - : compiler(compiler_) + : OpcodeHandler(compiler_) { } -bool Compiler::CFGBuilder::handle(spv::Op, const uint32_t *, uint32_t) +bool Compiler::CFGBuilder::handle(Op, const uint32_t *, uint32_t) { return true; } @@ -4762,13 +5069,16 @@ void Compiler::make_constant_null(uint32_t id, uint32_t type) uint32_t parent_id = ir.increase_bound_by(1); make_constant_null(parent_id, constant_type.parent_type); - if (!constant_type.array_size_literal.back()) - SPIRV_CROSS_THROW("Array size of OpConstantNull must be a literal."); - - SmallVector elements(constant_type.array.back()); - for (uint32_t i = 0; i < constant_type.array.back(); i++) + // The array size of OpConstantNull can be either literal or specialization constant. + // In the latter case, we cannot take the value as-is, as it can be changed to anything. + // Rather, we assume it to be *one* for the sake of initializer. + bool is_literal_array_size = constant_type.array_size_literal.back(); + uint32_t count = is_literal_array_size ? constant_type.array.back() : 1; + SmallVector elements(count); + for (uint32_t i = 0; i < count; i++) elements[i] = parent_id; - set(id, type, elements.data(), uint32_t(elements.size()), false); + auto &constant = set(id, type, elements.data(), uint32_t(elements.size()), false); + constant.is_null_array_specialized_length = !is_literal_array_size; } else if (!constant_type.member_types.empty()) { @@ -4788,7 +5098,7 @@ void Compiler::make_constant_null(uint32_t id, uint32_t type) } } -const SmallVector &Compiler::get_declared_capabilities() const +const SmallVector &Compiler::get_declared_capabilities() const { return ir.declared_capabilities; } @@ -4830,12 +5140,12 @@ std::string Compiler::get_remapped_declared_block_name(uint32_t id, bool fallbac bool Compiler::reflection_ssbo_instance_name_is_significant() const { - if (ir.source.known) + if (!ir.sources.empty() && ir.sources[0].known) { // UAVs from HLSL source tend to be declared in a way where the type is reused // but the instance name is significant, and that's the name we should report. // For GLSL, SSBOs each have their own block type as that's how GLSL is written. - return ir.source.hlsl; + return ir.sources[0].hlsl; } unordered_set ssbo_type_ids; @@ -4863,7 +5173,7 @@ bool Compiler::reflection_ssbo_instance_name_is_significant() const return aliased_ssbo_types; } -bool Compiler::instruction_to_result_type(uint32_t &result_type, uint32_t &result_id, spv::Op op, +bool Compiler::instruction_to_result_type(uint32_t &result_type, uint32_t &result_id, Op op, const uint32_t *args, uint32_t length) { if (length < 2) @@ -4910,7 +5220,7 @@ Bitset Compiler::combined_decoration_for_member(const SPIRType &type, uint32_t i return flags; } -bool Compiler::is_desktop_only_format(spv::ImageFormat format) +bool Compiler::is_desktop_only_format(ImageFormat format) { switch (format) { @@ -4953,7 +5263,7 @@ bool Compiler::is_depth_image(const SPIRType &type, uint32_t id) const bool Compiler::type_is_opaque_value(const SPIRType &type) const { return !type.pointer && (type.basetype == SPIRType::SampledImage || type.basetype == SPIRType::Image || - type.basetype == SPIRType::Sampler); + type.basetype == SPIRType::Sampler || type.basetype == SPIRType::Tensor); } // Make these member functions so we can easily break on any force_recompile events. @@ -4980,7 +5290,7 @@ void Compiler::clear_force_recompile() } Compiler::PhysicalStorageBufferPointerHandler::PhysicalStorageBufferPointerHandler(Compiler &compiler_) - : compiler(compiler_) + : OpcodeHandler(compiler_) { } @@ -5024,13 +5334,12 @@ void Compiler::PhysicalStorageBufferPointerHandler::mark_aligned_access(uint32_t bool Compiler::PhysicalStorageBufferPointerHandler::type_is_bda_block_entry(uint32_t type_id) const { auto &type = compiler.get(type_id); - return type.storage == StorageClassPhysicalStorageBufferEXT && type.pointer && - type.pointer_depth == 1 && !compiler.type_is_array_of_pointers(type); + return compiler.is_physical_pointer(type); } uint32_t Compiler::PhysicalStorageBufferPointerHandler::get_minimum_scalar_alignment(const SPIRType &type) const { - if (type.storage == spv::StorageClassPhysicalStorageBufferEXT) + if (type.storage == StorageClassPhysicalStorageBuffer) return 8; else if (type.basetype == SPIRType::Struct) { @@ -5055,7 +5364,8 @@ void Compiler::PhysicalStorageBufferPointerHandler::setup_meta_chain(uint32_t ty access_chain_to_physical_block[var_id] = &meta; auto &type = compiler.get(type_id); - if (type.basetype != SPIRType::Struct) + + if (!compiler.is_physical_pointer_to_buffer_block(type)) non_block_types.insert(type_id); if (meta.alignment == 0) @@ -5104,6 +5414,13 @@ bool Compiler::PhysicalStorageBufferPointerHandler::handle(Op op, const uint32_t break; } + case OpCooperativeMatrixLoadKHR: + case OpCooperativeMatrixStoreKHR: + { + // TODO: Can we meaningfully deal with this? + break; + } + default: break; } @@ -5114,9 +5431,7 @@ bool Compiler::PhysicalStorageBufferPointerHandler::handle(Op op, const uint32_t uint32_t Compiler::PhysicalStorageBufferPointerHandler::get_base_non_block_type_id(uint32_t type_id) const { auto *type = &compiler.get(type_id); - while (type->pointer && - type->storage == StorageClassPhysicalStorageBufferEXT && - !type_is_bda_block_entry(type_id)) + while (compiler.is_physical_pointer(*type) && !type_is_bda_block_entry(type_id)) { type_id = type->parent_type; type = &compiler.get(type_id); @@ -5128,15 +5443,17 @@ uint32_t Compiler::PhysicalStorageBufferPointerHandler::get_base_non_block_type_ void Compiler::PhysicalStorageBufferPointerHandler::analyze_non_block_types_from_block(const SPIRType &type) { + if (analyzed_type_ids.count(type.self)) + return; + analyzed_type_ids.insert(type.self); + for (auto &member : type.member_types) { auto &subtype = compiler.get(member); - if (subtype.basetype != SPIRType::Struct && subtype.pointer && - subtype.storage == spv::StorageClassPhysicalStorageBufferEXT) - { + + if (compiler.is_physical_pointer(subtype) && !compiler.is_physical_pointer_to_buffer_block(subtype)) non_block_types.insert(get_base_non_block_type_id(member)); - } - else if (subtype.basetype == SPIRType::Struct && !subtype.pointer) + else if (subtype.basetype == SPIRType::Struct && !compiler.is_pointer(subtype)) analyze_non_block_types_from_block(subtype); } } @@ -5149,9 +5466,14 @@ void Compiler::analyze_non_block_pointer_types() // Analyze any block declaration we have to make. It might contain // physical pointers to POD types which we never used, and thus never added to the list. // We'll need to add those pointer types to the set of types we declare. - ir.for_each_typed_id([&](uint32_t, SPIRType &type) { - if (has_decoration(type.self, DecorationBlock) || has_decoration(type.self, DecorationBufferBlock)) + ir.for_each_typed_id([&](uint32_t id, SPIRType &type) { + // Only analyze the raw block struct, not any pointer-to-struct, since that's just redundant. + if (type.self == id && + (has_decoration(type.self, DecorationBlock) || + has_decoration(type.self, DecorationBufferBlock))) + { handler.analyze_non_block_types_from_block(type); + } }); physical_storage_non_block_pointer_types.reserve(handler.non_block_types.size()); @@ -5161,6 +5483,212 @@ void Compiler::analyze_non_block_pointer_types() physical_storage_type_to_alignment = std::move(handler.physical_block_type_meta); } +void Compiler::analyze_descriptor_heap_types() +{ + struct HeapHandler : OpcodeHandler + { + bool handle(Op opcode, const uint32_t *args, uint32_t) override + { + switch (opcode) + { + case OpBufferPointerEXT: + { + auto &ptr_type = compiler.get(args[0]); + // BufferPointerEXT can return untyped or typed pointers. + // If it's typed, we resolve it here. + if (ptr_type.basetype == SPIRType::Struct) + { + DescriptorHeapMeta meta = {}; + meta.type = ptr_type.self; + meta.hlsl_style_stride = hlsl_style_stride_access_chains.count(args[2]); + meta.buffer_pointer_id = args[1]; + meta.storage = ptr_type.storage; + meta.nonreadable = compiler.has_decoration(args[1], DecorationNonReadable); + meta.nonwritable = compiler.has_decoration(args[1], DecorationNonWritable); + meta.coherent = compiler.has_decoration(args[1], DecorationCoherent); + meta.is_restrict = compiler.has_decoration(args[1], DecorationRestrict); + meta.is_volatile = compiler.has_decoration(args[1], DecorationVolatile); + add_unique_type(meta); + } + buffer_pointers[args[1]] = { args[0], hlsl_style_stride_access_chains.count(args[2]) != 0 }; + break; + } + + case OpUntypedAccessChainKHR: + case OpUntypedInBoundsAccessChainKHR: + case OpUntypedArrayLengthKHR: + { + auto &data_type = compiler.get(args[2]); + + if (compiler.is_pointer(data_type)) + SPIRV_CROSS_THROW("pointer type not allowed."); + + bool hlsl_style_stride = false; + + // Need to validate the array stride and types. HLLs are not flexible enough to support the full flexibility of SPIR-V. + if (BuiltIn(compiler.get_decoration(args[3], DecorationBuiltIn)) == BuiltInResourceHeapEXT) + { + if (!compiler.is_runtime_size_array(data_type)) + SPIRV_CROSS_THROW("Descriptor heap must be accessed as a runtime array."); + + // The only meaningful use of this is ArrayStride equal to sizeof(type) right now. + uint32_t array_stride_id = compiler.get_decoration(args[2], DecorationArrayStrideIdEXT); + if (!array_stride_id) + SPIRV_CROSS_THROW("Expected ArrayStrideIdEXT to be set for resource heap."); + + auto *spec_c = compiler.maybe_get(array_stride_id); + auto *c = compiler.maybe_get(array_stride_id); + + if (!spec_c && !c) + SPIRV_CROSS_THROW("Array stride must be some constant expression."); + + if (spec_c) + { + // This gets potentially infinitely weird, but if we get HLSL-style shaders + // we expect the array stride to be max(buffer, image) since all descriptors have equal size in D3D12. + // We just have to be a bit loose here since it's impossible to anticipate every theoretical formulation. + // Anything non-conforming to strict GLSL is flagged in the codegen output. + if (spec_c->opcode == OpSelect) + { + auto *true_value = compiler.maybe_get(spec_c->arguments[1]); + auto *false_value = compiler.maybe_get(spec_c->arguments[2]); + hlsl_style_stride = true_value && true_value->size_of_type && + false_value && false_value->size_of_type; + } + + if (!hlsl_style_stride) + SPIRV_CROSS_THROW("Unusual pattern of descriptor stride detected. This probably cannot be expressed in current GLSL."); + } + + if (c && !c->size_of_type) + SPIRV_CROSS_THROW("Resource heap array stride must be ConstantSizeOfEXT for high level languages."); + + auto &element_type = compiler.get(data_type.parent_type); + + if (element_type.basetype == SPIRType::DescriptorHeapBuffer) + { + if (c && compiler.get(c->size_of_type).basetype != SPIRType::DescriptorHeapBuffer) + SPIRV_CROSS_THROW("Buffer descriptors in heap must be ConstantSizeOfEXT(OpTypeBufferEXT) for GLSL."); + } + else if (data_type.basetype == SPIRType::Image) + { + if (c && compiler.get(c->size_of_type).basetype != SPIRType::Image) + SPIRV_CROSS_THROW("Image descriptors in heap must be ConstantSizeOfEXT(OpTypeImage) for GLSL."); + } + else if (data_type.basetype == SPIRType::AccelerationStructure) + { + if (c && compiler.get(c->size_of_type).basetype != SPIRType::AccelerationStructure) + SPIRV_CROSS_THROW("RTAS descriptors in heap must be ConstantSizeOfEXT(OpTypeAccelerationStructure) for GLSL."); + } + } + else if (BuiltIn(compiler.get_decoration(args[3], DecorationBuiltIn)) == BuiltInSamplerHeapEXT) + { + if (!compiler.is_runtime_size_array(data_type)) + SPIRV_CROSS_THROW("Descriptor heap must be accessed as a runtime array."); + + // The only meaningful use of this is ArrayStride equal to sizeof(sampler) right now. + uint32_t array_stride_id = compiler.get_decoration(args[2], DecorationArrayStrideIdEXT); + if (!array_stride_id) + SPIRV_CROSS_THROW("Expected ArrayStrideIdEXT to be set for sampler heap."); + + auto *c = compiler.maybe_get(array_stride_id); + if (!c || !c->size_of_type || compiler.get(c->size_of_type).basetype != SPIRType::Sampler) + SPIRV_CROSS_THROW("Sampler heap array stride must be ConstantSizeOfEXT(OpTypeSampler) for high level languages."); + } + + // Remember this for OpBufferPointerEXT. + if (hlsl_style_stride) + hlsl_style_stride_access_chains.insert(args[1]); + + if (data_type.basetype == SPIRType::SampledImage) + { + SPIRV_CROSS_THROW("Attempting to access heap as combined sampler image. This does not make sense."); + } + else if (data_type.basetype == SPIRType::Image || + data_type.basetype == SPIRType::AccelerationStructure || + data_type.basetype == SPIRType::Sampler) + { + DescriptorHeapMeta meta = {}; + meta.type = data_type.self; + meta.hlsl_style_stride = hlsl_style_stride; + add_unique_type(meta); + } + else if (buffer_pointers.count(args[3]) != 0) + { + if (!compiler.has_decoration(data_type.self, DecorationBlock) && + !compiler.has_decoration(data_type.self, DecorationBufferBlock)) + { + SPIRV_CROSS_THROW("BufferPointerEXT must reference a block type."); + } + + auto &pointer_meta = buffer_pointers[args[3]]; + auto &buffer_type = compiler.get(pointer_meta.type); + if (buffer_type.basetype == SPIRType::Void) + { + // This is where the pointer becomes typed, so register it here. + DescriptorHeapMeta meta = {}; + meta.type = data_type.self; + meta.hlsl_style_stride = pointer_meta.hlsl_style_stride; + meta.buffer_pointer_id = args[3]; + meta.storage = buffer_type.storage; + meta.nonreadable = compiler.has_decoration(args[3], DecorationNonReadable); + meta.nonwritable = compiler.has_decoration(args[3], DecorationNonWritable); + meta.coherent = compiler.has_decoration(args[3], DecorationCoherent); + meta.is_volatile = compiler.has_decoration(args[3], DecorationVolatile); + meta.is_restrict = compiler.has_decoration(args[3], DecorationRestrict); + add_unique_type(meta); + } + } + break; + } + + default: + break; + } + + return true; + } + + explicit HeapHandler(Compiler &compiler_) : OpcodeHandler(compiler_) {} + + std::vector heap_types; + + struct BufferPointerMeta + { + TypeID type; + bool hlsl_style_stride; + }; + std::unordered_map buffer_pointers; + std::unordered_set hlsl_style_stride_access_chains; + + void add_unique_type(const DescriptorHeapMeta &meta) + { + assert(meta.type != 0); + + for (auto &type : heap_types) + { + if (type.type == meta.type && type.storage == meta.storage && + type.buffer_pointer_id == meta.buffer_pointer_id && + type.nonreadable == meta.nonreadable && + type.nonwritable == meta.nonwritable && + type.coherent == meta.coherent && + type.is_restrict == meta.is_restrict && + type.hlsl_style_stride == meta.hlsl_style_stride && + type.is_volatile == meta.is_volatile) + { + return; + } + } + + heap_types.push_back(meta); + } + }; + + HeapHandler handler(*this); + traverse_all_reachable_opcodes(get(ir.default_entry_point), handler); + descriptor_heap_types = std::move(handler.heap_types); +} + bool Compiler::InterlockedResourceAccessPrepassHandler::handle(Op op, const uint32_t *, uint32_t) { if (op == OpBeginInvocationInterlockEXT || op == OpEndInvocationInterlockEXT) @@ -5258,6 +5786,8 @@ bool Compiler::InterlockedResourceAccessHandler::handle(Op opcode, const uint32_ switch (opcode) { case OpLoad: + case OpCooperativeMatrixLoadKHR: + case OpCooperativeVectorLoadNV: { if (length < 3) return false; @@ -5335,6 +5865,8 @@ bool Compiler::InterlockedResourceAccessHandler::handle(Op opcode, const uint32_ case OpStore: case OpImageWrite: case OpAtomicStore: + case OpCooperativeMatrixStoreKHR: + case OpCooperativeVectorStoreNV: { if (length < 1) return false; @@ -5531,3 +6063,12 @@ void Compiler::add_loop_level() { current_loop_level++; } + +const SPIRType *Compiler::OpcodeHandler::get_expression_result_type(uint32_t id) const +{ + auto itr = result_types.find(id); + if (itr == result_types.end()) + return nullptr; + + return &compiler.get(itr->second); +} diff --git a/src/libraries/spirv_cross/spirv_cross.hpp b/src/libraries/spirv_cross/spirv_cross.hpp index 9fe6c41c4..d8b05169f 100644 --- a/src/libraries/spirv_cross/spirv_cross.hpp +++ b/src/libraries/spirv_cross/spirv_cross.hpp @@ -27,12 +27,20 @@ #ifndef SPV_ENABLE_UTILITY_CODE #define SPV_ENABLE_UTILITY_CODE #endif + +// Pragmatic hack to avoid symbol conflicts when including both hpp11 and hpp headers in same translation unit. +// This is an unfortunate SPIRV-Headers issue that we cannot easily deal with ourselves. +#ifdef SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#define spv SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#endif + #include "spirv.hpp" #include "spirv_cfg.hpp" #include "spirv_cross_parsed_ir.hpp" namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; struct Resource { // Resources are identified with their SPIR-V ID. @@ -69,7 +77,7 @@ struct BuiltInResource // A builtin present here does not necessarily mean it's considered an active builtin, // since variable ID "activeness" is only tracked on OpVariable level, not Block members. // For that, update_active_builtins() -> has_active_builtin() can be used to further refine the reflection. - spv::BuiltIn builtin; + BuiltIn builtin; // This is the actual value type of the builtin. // Typically float4, float, array for the gl_PerVertex builtins. @@ -83,6 +91,28 @@ struct BuiltInResource Resource resource; }; +// Needs to stay in sync 1:1 with C API. +enum ResourceType +{ + ResourceTypeUnknown = 0, + ResourceTypeUniformBuffer = 1, + ResourceTypeStorageBuffer = 2, + ResourceTypeStageInput = 3, + ResourceTypeStageOutput = 4, + ResourceTypeSubpassInput = 5, + ResourceTypeStorageImage = 6, + ResourceTypeSampledImage = 7, + ResourceTypeAtomicCounter = 8, + ResourceTypePushConstant = 9, + ResourceTypeSeparateImage = 10, + ResourceTypeSeparateSamplers = 11, + ResourceTypeAccelerationStructure = 12, + ResourceTypeRayQuery = 13, + ResourceTypeShaderRecordBuffer = 14, + ResourceTypeGLPlainUniform = 15, + ResourceTypeTensor = 16 +}; + struct ShaderResources { SmallVector uniform_buffers; @@ -95,6 +125,7 @@ struct ShaderResources SmallVector atomic_counters; SmallVector acceleration_structures; SmallVector gl_plain_uniforms; + SmallVector tensors; // There can only be one push constant block, // but keep the vector in case this restriction is lifted in the future. @@ -151,7 +182,7 @@ enum BufferPackingStandard struct EntryPoint { std::string name; - spv::ExecutionModel execution_model; + ExecutionModel execution_model; }; class Compiler @@ -182,8 +213,8 @@ public: const std::string &get_name(ID id) const; // Applies a decoration to an ID. Effectively injects OpDecorate. - void set_decoration(ID id, spv::Decoration decoration, uint32_t argument = 0); - void set_decoration_string(ID id, spv::Decoration decoration, const std::string &argument); + void set_decoration(ID id, Decoration decoration, uint32_t argument = 0); + void set_decoration_string(ID id, Decoration decoration, const std::string &argument); // Overrides the identifier OpName of an ID. // Identifiers beginning with underscores or identifiers which contain double underscores @@ -191,22 +222,22 @@ public: void set_name(ID id, const std::string &name); // Gets a bitmask for the decorations which are applied to ID. - // I.e. (1ull << spv::DecorationFoo) | (1ull << spv::DecorationBar) + // I.e. (1ull << DecorationFoo) | (1ull << DecorationBar) const Bitset &get_decoration_bitset(ID id) const; // Returns whether the decoration has been applied to the ID. - bool has_decoration(ID id, spv::Decoration decoration) const; + bool has_decoration(ID id, Decoration decoration) const; // Gets the value for decorations which take arguments. - // If the decoration is a boolean (i.e. spv::DecorationNonWritable), + // If the decoration is a boolean (i.e. DecorationNonWritable), // 1 will be returned. // If decoration doesn't exist or decoration is not recognized, // 0 will be returned. - uint32_t get_decoration(ID id, spv::Decoration decoration) const; - const std::string &get_decoration_string(ID id, spv::Decoration decoration) const; + uint32_t get_decoration(ID id, Decoration decoration) const; + const std::string &get_decoration_string(ID id, Decoration decoration) const; // Removes the decoration for an ID. - void unset_decoration(ID id, spv::Decoration decoration); + void unset_decoration(ID id, Decoration decoration); // Gets the SPIR-V type associated with ID. // Mostly used with Resource::type_id and Resource::base_type_id to parse the underlying type of a resource. @@ -216,7 +247,7 @@ public: const SPIRType &get_type_from_variable(VariableID id) const; // Gets the underlying storage class for an OpVariable. - spv::StorageClass get_storage_class(VariableID id) const; + StorageClass get_storage_class(VariableID id) const; // If get_name() is an empty string, get the fallback name which will be used // instead in the disassembled source. @@ -231,8 +262,8 @@ public: const std::string &get_member_name(TypeID id, uint32_t index) const; // Given an OpTypeStruct in ID, obtain the OpMemberDecoration for member number "index". - uint32_t get_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration) const; - const std::string &get_member_decoration_string(TypeID id, uint32_t index, spv::Decoration decoration) const; + uint32_t get_member_decoration(TypeID id, uint32_t index, Decoration decoration) const; + const std::string &get_member_decoration_string(TypeID id, uint32_t index, Decoration decoration) const; // Sets the member identifier for OpTypeStruct ID, member number "index". void set_member_name(TypeID id, uint32_t index, const std::string &name); @@ -245,15 +276,15 @@ public: const Bitset &get_member_decoration_bitset(TypeID id, uint32_t index) const; // Returns whether the decoration has been applied to a member of a struct. - bool has_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration) const; + bool has_member_decoration(TypeID id, uint32_t index, Decoration decoration) const; // Similar to set_decoration, but for struct members. - void set_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration, uint32_t argument = 0); - void set_member_decoration_string(TypeID id, uint32_t index, spv::Decoration decoration, + void set_member_decoration(TypeID id, uint32_t index, Decoration decoration, uint32_t argument = 0); + void set_member_decoration_string(TypeID id, uint32_t index, Decoration decoration, const std::string &argument); // Unsets a member decoration, similar to unset_decoration. - void unset_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration); + void unset_member_decoration(TypeID id, uint32_t index, Decoration decoration); // Gets the fallback name for a member, similar to get_fallback_name. virtual const std::string get_fallback_member_name(uint32_t index) const @@ -339,28 +370,28 @@ public: // Names for entry points in the SPIR-V module may alias if they belong to different execution models. // To disambiguate, we must pass along with the entry point names the execution model. SmallVector get_entry_points_and_stages() const; - void set_entry_point(const std::string &entry, spv::ExecutionModel execution_model); + void set_entry_point(const std::string &entry, ExecutionModel execution_model); // Renames an entry point from old_name to new_name. // If old_name is currently selected as the current entry point, it will continue to be the current entry point, // albeit with a new name. // get_entry_points() is essentially invalidated at this point. void rename_entry_point(const std::string &old_name, const std::string &new_name, - spv::ExecutionModel execution_model); - const SPIREntryPoint &get_entry_point(const std::string &name, spv::ExecutionModel execution_model) const; - SPIREntryPoint &get_entry_point(const std::string &name, spv::ExecutionModel execution_model); + ExecutionModel execution_model); + const SPIREntryPoint &get_entry_point(const std::string &name, ExecutionModel execution_model) const; + SPIREntryPoint &get_entry_point(const std::string &name, ExecutionModel execution_model); const std::string &get_cleansed_entry_point_name(const std::string &name, - spv::ExecutionModel execution_model) const; + ExecutionModel execution_model) const; // Traverses all reachable opcodes and sets active_builtins to a bitmask of all builtin variables which are accessed in the shader. void update_active_builtins(); - bool has_active_builtin(spv::BuiltIn builtin, spv::StorageClass storage) const; + bool has_active_builtin(BuiltIn builtin, StorageClass storage) const; // Query and modify OpExecutionMode. const Bitset &get_execution_mode_bitset() const; - void unset_execution_mode(spv::ExecutionMode mode); - void set_execution_mode(spv::ExecutionMode mode, uint32_t arg0 = 0, uint32_t arg1 = 0, uint32_t arg2 = 0); + void unset_execution_mode(ExecutionMode mode); + void set_execution_mode(ExecutionMode mode, uint32_t arg0 = 0, uint32_t arg1 = 0, uint32_t arg2 = 0); // Gets argument for an execution mode (LocalSize, Invocations, OutputVertices). // For LocalSize or LocalSizeId, the index argument is used to select the dimension (X = 0, Y = 1, Z = 2). @@ -368,8 +399,8 @@ public: // LocalSizeId query returns an ID. If LocalSizeId execution mode is not used, it returns 0. // LocalSize always returns a literal. If execution mode is LocalSizeId, // the literal (spec constant or not) is still returned. - uint32_t get_execution_mode_argument(spv::ExecutionMode mode, uint32_t index = 0) const; - spv::ExecutionModel get_execution_model() const; + uint32_t get_execution_mode_argument(ExecutionMode mode, uint32_t index = 0) const; + ExecutionModel get_execution_model() const; bool is_tessellation_shader() const; bool is_tessellating_triangles() const; @@ -482,7 +513,7 @@ public: // If the decoration was declared, sets the word_offset to an offset into the provided SPIR-V binary buffer and returns true, // otherwise, returns false. // If the decoration does not have any value attached to it (e.g. DecorationRelaxedPrecision), this function will also return false. - bool get_binary_offset_for_decoration(VariableID id, spv::Decoration decoration, uint32_t &word_offset) const; + bool get_binary_offset_for_decoration(VariableID id, Decoration decoration, uint32_t &word_offset) const; // HLSL counter buffer reflection interface. // Append/Consume/Increment/Decrement in HLSL is implemented as two "neighbor" buffer objects where @@ -508,7 +539,7 @@ public: bool buffer_get_hlsl_counter_buffer(VariableID id, uint32_t &counter_id) const; // Gets the list of all SPIR-V Capabilities which were declared in the SPIR-V module. - const SmallVector &get_declared_capabilities() const; + const SmallVector &get_declared_capabilities() const; // Gets the list of all SPIR-V extensions which were declared in the SPIR-V module. const SmallVector &get_declared_extensions() const; @@ -540,6 +571,9 @@ public: return position_invariant; } + const ParsedIR &get_ir() const { return ir; } + uint32_t evaluate_constant_u32(uint32_t id) const; + protected: const uint32_t *stream(const Instruction &instr) const { @@ -573,6 +607,7 @@ protected: // (SSBO, image load store, etc) SmallVector global_variables; SmallVector aliased_variables; + SmallVector buffer_pointer_variables; SPIRFunction *current_function = nullptr; SPIRBlock *current_block = nullptr; @@ -671,26 +706,29 @@ protected: const SPIREntryPoint &get_entry_point() const; SPIREntryPoint &get_entry_point(); - static bool is_tessellation_shader(spv::ExecutionModel model); + static bool is_tessellation_shader(ExecutionModel model); virtual std::string to_name(uint32_t id, bool allow_alias = true) const; bool is_builtin_variable(const SPIRVariable &var) const; bool is_builtin_type(const SPIRType &type) const; bool is_hidden_variable(const SPIRVariable &var, bool include_builtins = false) const; bool is_immutable(uint32_t id) const; - bool is_member_builtin(const SPIRType &type, uint32_t index, spv::BuiltIn *builtin) const; + bool is_member_builtin(const SPIRType &type, uint32_t index, BuiltIn *builtin) const; bool is_scalar(const SPIRType &type) const; bool is_vector(const SPIRType &type) const; bool is_matrix(const SPIRType &type) const; bool is_array(const SPIRType &type) const; bool is_pointer(const SPIRType &type) const; bool is_physical_pointer(const SPIRType &type) const; + bool is_physical_or_buffer_pointer(const SPIRType &type) const; + bool is_physical_pointer_to_buffer_block(const SPIRType &type) const; static bool is_runtime_size_array(const SPIRType &type); uint32_t expression_type_id(uint32_t id) const; const SPIRType &expression_type(uint32_t id) const; bool expression_is_lvalue(uint32_t id) const; bool variable_storage_is_aliased(const SPIRVariable &var); SPIRVariable *maybe_get_backing_variable(uint32_t chain); + SPIRExpression *maybe_get_backing_buffer_pointer(uint32_t chain); void register_read(uint32_t expr, uint32_t chain, bool forwarded); void register_write(uint32_t chain); @@ -725,6 +763,7 @@ protected: // Dependency tracking for temporaries read from variables. void flush_dependees(SPIRVariable &var); + void flush_dependees(SPIRExpression &expr); void flush_all_active_variables(); void flush_control_dependent_expressions(uint32_t block); void flush_all_atomic_capable_variables(); @@ -743,6 +782,8 @@ protected: bool function_is_pure(const SPIRFunction &func); bool block_is_pure(const SPIRBlock &block); + bool function_is_control_dependent(const SPIRFunction &func); + bool block_is_control_dependent(const SPIRBlock &block); bool execution_is_branchless(const SPIRBlock &from, const SPIRBlock &to) const; bool execution_is_direct_branch(const SPIRBlock &from, const SPIRBlock &to) const; @@ -784,11 +825,12 @@ protected: // Used internally to implement various traversals for queries. struct OpcodeHandler { + explicit OpcodeHandler(Compiler &compiler_) : compiler(compiler_) {} virtual ~OpcodeHandler() = default; // Return true if traversal should continue. // If false, traversal will end immediately. - virtual bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) = 0; + virtual bool handle(Op opcode, const uint32_t *args, uint32_t length) = 0; virtual bool handle_terminator(const SPIRBlock &) { return true; @@ -819,20 +861,40 @@ protected: { return true; } + + Compiler &compiler; + std::unordered_map result_types; + const SPIRType *get_expression_result_type(uint32_t id) const; + bool enable_result_types = false; + + template T &get(uint32_t id) + { + return compiler.get(id); + } + + template const T &get(uint32_t id) const + { + return compiler.get(id); + } + + template + T &set(uint32_t id, P &&... args) + { + return compiler.set(id, std::forward

(args)...); + } }; struct BufferAccessHandler : OpcodeHandler { BufferAccessHandler(const Compiler &compiler_, SmallVector &ranges_, uint32_t id_) - : compiler(compiler_) + : OpcodeHandler(const_cast(compiler_)) , ranges(ranges_) , id(id_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; - const Compiler &compiler; SmallVector &ranges; uint32_t id; @@ -842,29 +904,26 @@ protected: struct InterfaceVariableAccessHandler : OpcodeHandler { InterfaceVariableAccessHandler(const Compiler &compiler_, std::unordered_set &variables_) - : compiler(compiler_) + : OpcodeHandler(const_cast(compiler_)) , variables(variables_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; - const Compiler &compiler; std::unordered_set &variables; }; struct CombinedImageSamplerHandler : OpcodeHandler { - CombinedImageSamplerHandler(Compiler &compiler_) - : compiler(compiler_) + explicit CombinedImageSamplerHandler(Compiler &compiler_) + : OpcodeHandler(compiler_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; bool begin_function_scope(const uint32_t *args, uint32_t length) override; bool end_function_scope(const uint32_t *args, uint32_t length) override; - Compiler &compiler; - // Each function in the call stack needs its own remapping for parameters so we can deduce which global variable each texture/sampler the parameter is statically bound to. std::stack> parameter_remapping; std::stack functions; @@ -878,27 +937,24 @@ protected: struct DummySamplerForCombinedImageHandler : OpcodeHandler { - DummySamplerForCombinedImageHandler(Compiler &compiler_) - : compiler(compiler_) + explicit DummySamplerForCombinedImageHandler(Compiler &compiler_) + : OpcodeHandler(compiler_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; - - Compiler &compiler; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; bool need_dummy_sampler = false; }; struct ActiveBuiltinHandler : OpcodeHandler { - ActiveBuiltinHandler(Compiler &compiler_) - : compiler(compiler_) + explicit ActiveBuiltinHandler(Compiler &compiler_) + : OpcodeHandler(compiler_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; - Compiler &compiler; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; - void handle_builtin(const SPIRType &type, spv::BuiltIn builtin, const Bitset &decoration_flags); + void handle_builtin(const SPIRType &type, BuiltIn builtin, const Bitset &decoration_flags); void add_if_builtin(uint32_t id); void add_if_builtin_or_block(uint32_t id); void add_if_builtin(uint32_t id, bool allow_blocks); @@ -950,13 +1006,12 @@ protected: struct CombinedImageSamplerDrefHandler : OpcodeHandler { - CombinedImageSamplerDrefHandler(Compiler &compiler_) - : compiler(compiler_) + explicit CombinedImageSamplerDrefHandler(Compiler &compiler_) + : OpcodeHandler(compiler_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; - Compiler &compiler; std::unordered_set dref_combined_samplers; }; @@ -964,14 +1019,13 @@ protected: { CombinedImageSamplerUsageHandler(Compiler &compiler_, const std::unordered_set &dref_combined_samplers_) - : compiler(compiler_) + : OpcodeHandler(compiler_) , dref_combined_samplers(dref_combined_samplers_) { } bool begin_function_scope(const uint32_t *args, uint32_t length) override; - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; - Compiler &compiler; + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; const std::unordered_set &dref_combined_samplers; std::unordered_map> dependency_hierarchy; @@ -993,8 +1047,7 @@ protected: explicit CFGBuilder(Compiler &compiler_); bool follow_function_call(const SPIRFunction &func) override; - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; - Compiler &compiler; + bool handle(Op op, const uint32_t *args, uint32_t length) override; std::unordered_map> function_cfgs; }; @@ -1008,10 +1061,9 @@ protected: void notify_variable_access(uint32_t id, uint32_t block); bool id_is_phi_variable(uint32_t id) const; bool id_is_potential_temporary(uint32_t id) const; - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; + bool handle(Op op, const uint32_t *args, uint32_t length) override; bool handle_terminator(const SPIRBlock &block) override; - Compiler &compiler; SPIRFunction &entry; std::unordered_map> accessed_variables_to_block; std::unordered_map> accessed_temporaries_to_block; @@ -1029,9 +1081,8 @@ protected: { StaticExpressionAccessHandler(Compiler &compiler_, uint32_t variable_id_); bool follow_function_call(const SPIRFunction &) override; - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; + bool handle(Op op, const uint32_t *args, uint32_t length) override; - Compiler &compiler; uint32_t variable_id; uint32_t static_expression = 0; uint32_t write_count = 0; @@ -1045,12 +1096,12 @@ protected: struct PhysicalStorageBufferPointerHandler : OpcodeHandler { explicit PhysicalStorageBufferPointerHandler(Compiler &compiler_); - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; - Compiler &compiler; + bool handle(Op op, const uint32_t *args, uint32_t length) override; std::unordered_set non_block_types; std::unordered_map physical_block_type_meta; std::unordered_map access_chain_to_physical_block; + std::unordered_set analyzed_type_ids; void mark_aligned_access(uint32_t id, const uint32_t *args, uint32_t length); PhysicalBlockMeta *find_block_meta(uint32_t id) const; @@ -1064,11 +1115,43 @@ protected: SmallVector physical_storage_non_block_pointer_types; std::unordered_map physical_storage_type_to_alignment; + struct DescriptorHeapMeta + { + TypeID type; + bool hlsl_style_stride; + + // For buffers + ID buffer_pointer_id; + StorageClass storage; + bool nonwritable; + bool nonreadable; + bool coherent; + bool is_volatile; + bool is_restrict; + }; + std::vector descriptor_heap_types; + void analyze_descriptor_heap_types(); + void analyze_variable_scope(SPIRFunction &function, AnalyzeVariableScopeAccessHandler &handler); void find_function_local_luts(SPIRFunction &function, const AnalyzeVariableScopeAccessHandler &handler, bool single_function); bool may_read_undefined_variable_in_block(const SPIRBlock &block, uint32_t var); + struct GeometryEmitDisocveryHandler : OpcodeHandler + { + explicit GeometryEmitDisocveryHandler(Compiler &compiler_) + : OpcodeHandler(compiler_) + { + } + + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; + bool begin_function_scope(const uint32_t *, uint32_t) override; + bool end_function_scope(const uint32_t *, uint32_t) override; + SmallVector function_stack; + }; + + void discover_geometry_emitters(); + // Finds all resources that are written to from inside the critical section, if present. // The critical section is delimited by OpBeginInvocationInterlockEXT and // OpEndInvocationInterlockEXT instructions. In MSL and HLSL, any resources written @@ -1076,16 +1159,15 @@ protected: struct InterlockedResourceAccessHandler : OpcodeHandler { InterlockedResourceAccessHandler(Compiler &compiler_, uint32_t entry_point_id) - : compiler(compiler_) + : OpcodeHandler(compiler_) { call_stack.push_back(entry_point_id); } - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; + bool handle(Op op, const uint32_t *args, uint32_t length) override; bool begin_function_scope(const uint32_t *args, uint32_t length) override; bool end_function_scope(const uint32_t *args, uint32_t length) override; - Compiler &compiler; bool in_crit_sec = false; uint32_t interlock_function_id = 0; @@ -1101,17 +1183,16 @@ protected: struct InterlockedResourceAccessPrepassHandler : OpcodeHandler { InterlockedResourceAccessPrepassHandler(Compiler &compiler_, uint32_t entry_point_id) - : compiler(compiler_) + : OpcodeHandler(compiler_) { call_stack.push_back(entry_point_id); } void rearm_current_block(const SPIRBlock &block) override; - bool handle(spv::Op op, const uint32_t *args, uint32_t length) override; + bool handle(Op op, const uint32_t *args, uint32_t length) override; bool begin_function_scope(const uint32_t *args, uint32_t length) override; bool end_function_scope(const uint32_t *args, uint32_t length) override; - Compiler &compiler; uint32_t interlock_function_id = 0; uint32_t current_block_id = 0; bool split_function_case = false; @@ -1128,11 +1209,11 @@ protected: std::unordered_map declared_block_names; - bool instruction_to_result_type(uint32_t &result_type, uint32_t &result_id, spv::Op op, const uint32_t *args, - uint32_t length); + static bool instruction_to_result_type( + uint32_t &result_type, uint32_t &result_id, Op op, const uint32_t *args, uint32_t length); Bitset combined_decoration_for_member(const SPIRType &type, uint32_t index) const; - static bool is_desktop_only_format(spv::ImageFormat format); + static bool is_desktop_only_format(ImageFormat format); bool is_depth_image(const SPIRType &type, uint32_t id) const; @@ -1151,6 +1232,7 @@ protected: bool type_contains_recursion(const SPIRType &type); bool type_is_array_of_pointers(const SPIRType &type) const; bool type_is_block_like(const SPIRType &type) const; + bool type_is_explicit_layout(const SPIRType &type) const; bool type_is_top_level_block(const SPIRType &type) const; bool type_is_opaque_value(const SPIRType &type) const; @@ -1160,7 +1242,6 @@ protected: bool flush_phi_required(BlockID from, BlockID to) const; uint32_t evaluate_spec_constant_u32(const SPIRConstantOp &spec) const; - uint32_t evaluate_constant_u32(uint32_t id) const; bool is_vertex_like_shader() const; @@ -1176,4 +1257,8 @@ private: }; } // namespace SPIRV_CROSS_NAMESPACE +#ifdef SPIRV_CROSS_SPV_HEADER_NAMESPACE_OVERRIDE +#undef spv +#endif + #endif diff --git a/src/libraries/spirv_cross/spirv_cross_containers.hpp b/src/libraries/spirv_cross/spirv_cross_containers.hpp index e79b32093..c496cb75b 100644 --- a/src/libraries/spirv_cross/spirv_cross_containers.hpp +++ b/src/libraries/spirv_cross/spirv_cross_containers.hpp @@ -576,6 +576,7 @@ public: if (!ptr) return nullptr; + vacants.reserve(num_objects); for (unsigned i = 0; i < num_objects; i++) vacants.push_back(&ptr[i]); diff --git a/src/libraries/spirv_cross/spirv_cross_error_handling.hpp b/src/libraries/spirv_cross/spirv_cross_error_handling.hpp index e96ebb9a7..91e6cf4f8 100644 --- a/src/libraries/spirv_cross/spirv_cross_error_handling.hpp +++ b/src/libraries/spirv_cross/spirv_cross_error_handling.hpp @@ -66,6 +66,11 @@ public: : std::runtime_error(str) { } + + explicit CompilerError(const char *str) + : std::runtime_error(str) + { + } }; #define SPIRV_CROSS_THROW(x) throw CompilerError(x) diff --git a/src/libraries/spirv_cross/spirv_cross_parsed_ir.cpp b/src/libraries/spirv_cross/spirv_cross_parsed_ir.cpp index c6ddb6a45..47cd1dbe1 100644 --- a/src/libraries/spirv_cross/spirv_cross_parsed_ir.cpp +++ b/src/libraries/spirv_cross/spirv_cross_parsed_ir.cpp @@ -26,7 +26,7 @@ #include using namespace std; -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; namespace SPIRV_CROSS_NAMESPACE { @@ -49,6 +49,7 @@ ParsedIR::ParsedIR() pool_group->pools[TypeAccessChain].reset(new ObjectPool); pool_group->pools[TypeUndef].reset(new ObjectPool); pool_group->pools[TypeString].reset(new ObjectPool); + pool_group->pools[TypeDebugLocalVariable].reset(new ObjectPool); } // Should have been default-implemented, but need this on MSVC 2013. @@ -78,7 +79,10 @@ ParsedIR &ParsedIR::operator=(ParsedIR &&other) SPIRV_CROSS_NOEXCEPT memory_model = other.memory_model; default_entry_point = other.default_entry_point; - source = other.source; + is_library_module = other.is_library_module; + library_exports = std::move(other.library_exports); + library_exported_functions = std::move(other.library_exported_functions); + sources = std::move(other.sources); loop_iteration_depth_hard = other.loop_iteration_depth_hard; loop_iteration_depth_soft = other.loop_iteration_depth_soft; @@ -110,7 +114,10 @@ ParsedIR &ParsedIR::operator=(const ParsedIR &other) continue_block_to_loop_header = other.continue_block_to_loop_header; entry_points = other.entry_points; default_entry_point = other.default_entry_point; - source = other.source; + is_library_module = other.is_library_module; + library_exports = other.library_exports; + library_exported_functions = other.library_exported_functions; + sources = other.sources; loop_iteration_depth_hard = other.loop_iteration_depth_hard; loop_iteration_depth_soft = other.loop_iteration_depth_soft; addressing_model = other.addressing_model; @@ -366,8 +373,8 @@ void ParsedIR::set_decoration_string(ID id, Decoration decoration, const string switch (decoration) { - case DecorationHlslSemanticGOOGLE: - dec.hlsl_semantic = argument; + case DecorationUserSemantic: + dec.user_semantic = argument; break; case DecorationUserTypeGOOGLE: @@ -403,6 +410,10 @@ void ParsedIR::set_decoration(ID id, Decoration decoration, uint32_t argument) dec.offset = argument; break; + case DecorationOffsetIdEXT: + dec.offset_id = argument; + break; + case DecorationXfbBuffer: dec.xfb_buffer = argument; break; @@ -419,6 +430,10 @@ void ParsedIR::set_decoration(ID id, Decoration decoration, uint32_t argument) dec.array_stride = argument; break; + case DecorationArrayStrideIdEXT: + dec.array_stride_id = argument; + break; + case DecorationMatrixStride: dec.matrix_stride = argument; break; @@ -452,6 +467,10 @@ void ParsedIR::set_decoration(ID id, Decoration decoration, uint32_t argument) dec.fp_rounding_mode = static_cast(argument); break; + case DecorationFPFastMathMode: + dec.fp_fast_math_mode = static_cast(argument); + break; + default: break; } @@ -487,6 +506,10 @@ void ParsedIR::set_member_decoration(TypeID id, uint32_t index, Decoration decor dec.offset = argument; break; + case DecorationOffsetIdEXT: + dec.offset_id = argument; + break; + case DecorationXfbBuffer: dec.xfb_buffer = argument; break; @@ -523,8 +546,27 @@ void ParsedIR::mark_used_as_array_length(ID id) switch (ids[id].get_type()) { case TypeConstant: - get(id).is_used_as_array_length = true; + { + auto &c = get(id); + c.is_used_as_array_length = true; + + // Mark composite dependencies as well. + for (auto &sub_id: c.m.id) + if (sub_id) + mark_used_as_array_length(sub_id); + + for (uint32_t col = 0; col < c.m.columns; col++) + { + for (auto &sub_id : c.m.c[col].id) + if (sub_id) + mark_used_as_array_length(sub_id); + } + + for (auto &sub_id : c.subconstants) + if (sub_id) + mark_used_as_array_length(sub_id); break; + } case TypeConstantOp: { @@ -564,7 +606,8 @@ Bitset ParsedIR::get_buffer_block_type_flags(const SPIRType &type) const Bitset ParsedIR::get_buffer_block_flags(const SPIRVariable &var) const { auto &type = get(var.basetype); - assert(type.basetype == SPIRType::Struct); + if (type.basetype != SPIRType::Struct) + SPIRV_CROSS_THROW("Cannot get buffer block flags for non-buffer variable."); // Some flags like non-writable, non-readable are actually found // as member decorations. If all members have a decoration set, propagate @@ -620,6 +663,8 @@ uint32_t ParsedIR::get_decoration(ID id, Decoration decoration) const return dec.component; case DecorationOffset: return dec.offset; + case DecorationOffsetIdEXT: + return dec.offset_id; case DecorationXfbBuffer: return dec.xfb_buffer; case DecorationXfbStride: @@ -636,12 +681,16 @@ uint32_t ParsedIR::get_decoration(ID id, Decoration decoration) const return dec.spec_id; case DecorationArrayStride: return dec.array_stride; + case DecorationArrayStrideIdEXT: + return dec.array_stride_id; case DecorationMatrixStride: return dec.matrix_stride; case DecorationIndex: return dec.index; case DecorationFPRoundingMode: return dec.fp_rounding_mode; + case DecorationFPFastMathMode: + return dec.fp_fast_math_mode; default: return 1; } @@ -660,8 +709,8 @@ const string &ParsedIR::get_decoration_string(ID id, Decoration decoration) cons switch (decoration) { - case DecorationHlslSemanticGOOGLE: - return dec.hlsl_semantic; + case DecorationUserSemantic: + return dec.user_semantic; case DecorationUserTypeGOOGLE: return dec.user_type; @@ -693,6 +742,10 @@ void ParsedIR::unset_decoration(ID id, Decoration decoration) dec.offset = 0; break; + case DecorationOffsetIdEXT: + dec.offset_id = 0; + break; + case DecorationXfbBuffer: dec.xfb_buffer = 0; break; @@ -721,14 +774,18 @@ void ParsedIR::unset_decoration(ID id, Decoration decoration) dec.spec_id = 0; break; - case DecorationHlslSemanticGOOGLE: - dec.hlsl_semantic.clear(); + case DecorationUserSemantic: + dec.user_semantic.clear(); break; case DecorationFPRoundingMode: dec.fp_rounding_mode = FPRoundingModeMax; break; + case DecorationFPFastMathMode: + dec.fp_fast_math_mode = FPFastMathModeMaskNone; + break; + case DecorationHlslCounterBufferGOOGLE: { auto &counter = meta[id].hlsl_magic_counter_buffer; @@ -775,6 +832,8 @@ uint32_t ParsedIR::get_member_decoration(TypeID id, uint32_t index, Decoration d return dec.binding; case DecorationOffset: return dec.offset; + case DecorationOffsetIdEXT: + return dec.offset_id; case DecorationXfbBuffer: return dec.xfb_buffer; case DecorationXfbStride: @@ -783,6 +842,8 @@ uint32_t ParsedIR::get_member_decoration(TypeID id, uint32_t index, Decoration d return dec.stream; case DecorationSpecId: return dec.spec_id; + case DecorationMatrixStride: + return dec.matrix_stride; case DecorationIndex: return dec.index; default: @@ -811,8 +872,8 @@ void ParsedIR::set_member_decoration_string(TypeID id, uint32_t index, Decoratio switch (decoration) { - case DecorationHlslSemanticGOOGLE: - dec.hlsl_semantic = argument; + case DecorationUserSemantic: + dec.user_semantic = argument; break; default: @@ -832,8 +893,8 @@ const string &ParsedIR::get_member_decoration_string(TypeID id, uint32_t index, switch (decoration) { - case DecorationHlslSemanticGOOGLE: - return dec.hlsl_semantic; + case DecorationUserSemantic: + return dec.user_semantic; default: return empty_string; @@ -870,6 +931,10 @@ void ParsedIR::unset_member_decoration(TypeID id, uint32_t index, Decoration dec dec.offset = 0; break; + case DecorationOffsetIdEXT: + dec.offset_id = 0; + break; + case DecorationXfbBuffer: dec.xfb_buffer = 0; break; @@ -886,8 +951,8 @@ void ParsedIR::unset_member_decoration(TypeID id, uint32_t index, Decoration dec dec.spec_id = 0; break; - case DecorationHlslSemanticGOOGLE: - dec.hlsl_semantic.clear(); + case DecorationUserSemantic: + dec.user_semantic.clear(); break; default: @@ -925,6 +990,8 @@ void ParsedIR::reset_all_of_type(Types type) void ParsedIR::add_typed_id(Types type, ID id) { + assert(id < ids.size()); + if (loop_iteration_depth_hard != 0) SPIRV_CROSS_THROW("Cannot add typed ID while looping over it."); @@ -1027,6 +1094,8 @@ ParsedIR::LoopLock &ParsedIR::LoopLock::operator=(LoopLock &&other) SPIRV_CROSS_ void ParsedIR::make_constant_null(uint32_t id, uint32_t type, bool add_to_typed_id_set) { + assert(id < ids.size()); + auto &constant_type = get(type); if (constant_type.pointer) @@ -1043,16 +1112,21 @@ void ParsedIR::make_constant_null(uint32_t id, uint32_t type, bool add_to_typed_ uint32_t parent_id = increase_bound_by(1); make_constant_null(parent_id, constant_type.parent_type, add_to_typed_id_set); - if (!constant_type.array_size_literal.back()) - SPIRV_CROSS_THROW("Array size of OpConstantNull must be a literal."); + // The array size of OpConstantNull can be either literal or specialization constant. + // In the latter case, we cannot take the value as-is, as it can be changed to anything. + // Rather, we assume it to be *one* for the sake of initializer. + bool is_literal_array_size = constant_type.array_size_literal.back(); + uint32_t count = is_literal_array_size ? constant_type.array.back() : 1; - SmallVector elements(constant_type.array.back()); - for (uint32_t i = 0; i < constant_type.array.back(); i++) + SmallVector elements(count); + for (uint32_t i = 0; i < count; i++) elements[i] = parent_id; if (add_to_typed_id_set) add_typed_id(TypeConstant, id); - variant_set(ids[id], type, elements.data(), uint32_t(elements.size()), false).self = id; + auto& constant = variant_set(ids[id], type, elements.data(), uint32_t(elements.size()), false); + constant.self = id; + constant.is_null_array_specialized_length = !is_literal_array_size; } else if (!constant_type.member_types.empty()) { diff --git a/src/libraries/spirv_cross/spirv_cross_parsed_ir.hpp b/src/libraries/spirv_cross/spirv_cross_parsed_ir.hpp index 7f35c3815..efe9d4a4b 100644 --- a/src/libraries/spirv_cross/spirv_cross_parsed_ir.hpp +++ b/src/libraries/spirv_cross/spirv_cross_parsed_ir.hpp @@ -30,6 +30,7 @@ namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; // This data structure holds all information needed to perform cross-compilation and reflection. // It is the output of the Parser, but any implementation could create this structure. @@ -87,7 +88,7 @@ public: // Declared capabilities and extensions in the SPIR-V module. // Not really used except for reflection at the moment. - SmallVector declared_capabilities; + SmallVector declared_capabilities; SmallVector declared_extensions; // Meta data about blocks. The cross-compiler needs to query if a block is either of these types. @@ -109,20 +110,43 @@ public: std::unordered_map entry_points; FunctionID default_entry_point = 0; + // A "library" module has no OpEntryPoint and instead exports symbols via + // OpDecorate ... LinkageAttributes ... Export. These vectors keep track + // of all these exports and specifically the function exports. + bool is_library_module = false; + SmallVector library_exports; + SmallVector library_exported_functions; + struct Source { + SourceLanguage lang = SourceLanguageUnknown; uint32_t version = 0; bool es = false; bool known = false; bool hlsl = false; + ID file_id = 0; // string + ID define_id = 0; // only non-zero for DebugSource + std::string source; + + struct Marker + { + ID line; // in source + ID col; // in source + ID offset; // in spirv stream + ID function_id; + ID block_id; + }; + + SmallVector line_markers; // sorted by line + Source() = default; }; - Source source; + std::vector sources; - spv::AddressingModel addressing_model = spv::AddressingModelMax; - spv::MemoryModel memory_model = spv::MemoryModelMax; + AddressingModel addressing_model = AddressingModelMax; + MemoryModel memory_model = MemoryModelMax; // Decoration handling methods. // Can be useful for simple "raw" reflection. @@ -130,25 +154,25 @@ public: // and might as well just have the whole suite of decoration/name handling in one place. void set_name(ID id, const std::string &name); const std::string &get_name(ID id) const; - void set_decoration(ID id, spv::Decoration decoration, uint32_t argument = 0); - void set_decoration_string(ID id, spv::Decoration decoration, const std::string &argument); - bool has_decoration(ID id, spv::Decoration decoration) const; - uint32_t get_decoration(ID id, spv::Decoration decoration) const; - const std::string &get_decoration_string(ID id, spv::Decoration decoration) const; + void set_decoration(ID id, Decoration decoration, uint32_t argument = 0); + void set_decoration_string(ID id, Decoration decoration, const std::string &argument); + bool has_decoration(ID id, Decoration decoration) const; + uint32_t get_decoration(ID id, Decoration decoration) const; + const std::string &get_decoration_string(ID id, Decoration decoration) const; const Bitset &get_decoration_bitset(ID id) const; - void unset_decoration(ID id, spv::Decoration decoration); + void unset_decoration(ID id, Decoration decoration); // Decoration handling methods (for members of a struct). void set_member_name(TypeID id, uint32_t index, const std::string &name); const std::string &get_member_name(TypeID id, uint32_t index) const; - void set_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration, uint32_t argument = 0); - void set_member_decoration_string(TypeID id, uint32_t index, spv::Decoration decoration, + void set_member_decoration(TypeID id, uint32_t index, Decoration decoration, uint32_t argument = 0); + void set_member_decoration_string(TypeID id, uint32_t index, Decoration decoration, const std::string &argument); - uint32_t get_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration) const; - const std::string &get_member_decoration_string(TypeID id, uint32_t index, spv::Decoration decoration) const; - bool has_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration) const; + uint32_t get_member_decoration(TypeID id, uint32_t index, Decoration decoration) const; + const std::string &get_member_decoration_string(TypeID id, uint32_t index, Decoration decoration) const; + bool has_member_decoration(TypeID id, uint32_t index, Decoration decoration) const; const Bitset &get_member_decoration_bitset(TypeID id, uint32_t index) const; - void unset_member_decoration(TypeID id, uint32_t index, spv::Decoration decoration); + void unset_member_decoration(TypeID id, uint32_t index, Decoration decoration); void mark_used_as_array_length(ID id); uint32_t increase_bound_by(uint32_t count); @@ -169,7 +193,7 @@ public: ~LoopLock(); private: - uint32_t *lock; + uint32_t *lock = nullptr; }; // This must be held while iterating over a type ID array. @@ -231,7 +255,6 @@ public: uint32_t get_spirv_version() const; -private: template T &get(uint32_t id) { @@ -244,6 +267,7 @@ private: return variant_get(ids[id]); } +private: mutable uint32_t loop_iteration_depth_hard = 0; mutable uint32_t loop_iteration_depth_soft = 0; std::string empty_string; diff --git a/src/libraries/spirv_cross/spirv_cross_util.cpp b/src/libraries/spirv_cross/spirv_cross_util.cpp index 7cff010d1..f30706f59 100644 --- a/src/libraries/spirv_cross/spirv_cross_util.cpp +++ b/src/libraries/spirv_cross/spirv_cross_util.cpp @@ -24,7 +24,7 @@ #include "spirv_cross_util.hpp" #include "spirv_common.hpp" -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; namespace spirv_cross_util @@ -34,10 +34,10 @@ void rename_interface_variable(Compiler &compiler, const SmallVector & { for (auto &v : resources) { - if (!compiler.has_decoration(v.id, spv::DecorationLocation)) + if (!compiler.has_decoration(v.id, DecorationLocation)) continue; - auto loc = compiler.get_decoration(v.id, spv::DecorationLocation); + auto loc = compiler.get_decoration(v.id, DecorationLocation); if (loc != location) continue; @@ -61,16 +61,16 @@ void inherit_combined_sampler_bindings(Compiler &compiler) auto &samplers = compiler.get_combined_image_samplers(); for (auto &s : samplers) { - if (compiler.has_decoration(s.image_id, spv::DecorationDescriptorSet)) + if (compiler.has_decoration(s.image_id, DecorationDescriptorSet)) { - uint32_t set = compiler.get_decoration(s.image_id, spv::DecorationDescriptorSet); - compiler.set_decoration(s.combined_id, spv::DecorationDescriptorSet, set); + uint32_t set = compiler.get_decoration(s.image_id, DecorationDescriptorSet); + compiler.set_decoration(s.combined_id, DecorationDescriptorSet, set); } - if (compiler.has_decoration(s.image_id, spv::DecorationBinding)) + if (compiler.has_decoration(s.image_id, DecorationBinding)) { - uint32_t binding = compiler.get_decoration(s.image_id, spv::DecorationBinding); - compiler.set_decoration(s.combined_id, spv::DecorationBinding, binding); + uint32_t binding = compiler.get_decoration(s.image_id, DecorationBinding); + compiler.set_decoration(s.combined_id, DecorationBinding, binding); } } } diff --git a/src/libraries/spirv_cross/spirv_glsl.cpp b/src/libraries/spirv_cross/spirv_glsl.cpp index f200b424d..cbd135d05 100644 --- a/src/libraries/spirv_cross/spirv_glsl.cpp +++ b/src/libraries/spirv_cross/spirv_glsl.cpp @@ -37,10 +37,13 @@ #endif #include -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; using namespace std; +namespace SPIRV_CROSS_NAMESPACE +{ + enum ExtraSubExpressionType { // Create masks above any legal ID range to allow multiple address spaces into the extra_sub_expressions map. @@ -48,6 +51,46 @@ enum ExtraSubExpressionType EXTRA_SUB_EXPRESSION_TYPE_AUX = 0x20000000 }; +struct GlslConstantNameMapping +{ + uint32_t value; + const char *alias; +}; + +#define DEF_GLSL_MAPPING(x) { x, "gl_" #x } +#define DEF_GLSL_MAPPING_EXT(x) { x##KHR, "gl_" #x } +static const GlslConstantNameMapping CoopVecComponentTypeNames[] = { + DEF_GLSL_MAPPING(ComponentTypeFloat16NV), + DEF_GLSL_MAPPING(ComponentTypeFloat32NV), + DEF_GLSL_MAPPING(ComponentTypeFloat64NV), + DEF_GLSL_MAPPING(ComponentTypeSignedInt8NV), + DEF_GLSL_MAPPING(ComponentTypeSignedInt16NV), + DEF_GLSL_MAPPING(ComponentTypeSignedInt32NV), + DEF_GLSL_MAPPING(ComponentTypeSignedInt64NV), + DEF_GLSL_MAPPING(ComponentTypeUnsignedInt8NV), + DEF_GLSL_MAPPING(ComponentTypeUnsignedInt16NV), + DEF_GLSL_MAPPING(ComponentTypeUnsignedInt32NV), + DEF_GLSL_MAPPING(ComponentTypeUnsignedInt64NV), + DEF_GLSL_MAPPING(ComponentTypeSignedInt8PackedNV), + DEF_GLSL_MAPPING(ComponentTypeUnsignedInt8PackedNV), + DEF_GLSL_MAPPING(ComponentTypeFloatE4M3NV), + DEF_GLSL_MAPPING(ComponentTypeFloatE5M2NV), +}; + +static const GlslConstantNameMapping CoopVecMatrixLayoutNames[] = { + DEF_GLSL_MAPPING(CooperativeVectorMatrixLayoutRowMajorNV), + DEF_GLSL_MAPPING(CooperativeVectorMatrixLayoutColumnMajorNV), + DEF_GLSL_MAPPING(CooperativeVectorMatrixLayoutInferencingOptimalNV), + DEF_GLSL_MAPPING(CooperativeVectorMatrixLayoutTrainingOptimalNV), +}; + +static const GlslConstantNameMapping CoopMatMatrixLayoutNames[] = { + DEF_GLSL_MAPPING_EXT(CooperativeMatrixLayoutRowMajor), + DEF_GLSL_MAPPING_EXT(CooperativeMatrixLayoutColumnMajor), +}; +#undef DEF_GLSL_MAPPING +#undef DEF_GLSL_MAPPING_EXT + static bool is_unsigned_opcode(Op op) { // Don't have to be exhaustive, only relevant for legacy target checking ... @@ -159,13 +202,14 @@ static BufferPackingStandard packing_to_substruct_packing(BufferPackingStandard return packing; } } +} void CompilerGLSL::init() { - if (ir.source.known) + if (!ir.sources.empty() && ir.sources.front().known) { - options.es = ir.source.es; - options.version = ir.source.version; + options.es = ir.sources.front().es; + options.version = ir.sources.front().version; } // Query the locale to see what the decimal point is. @@ -223,7 +267,7 @@ static const char *to_pls_layout(PlsFormat format) } } -static std::pair pls_format_to_basetype(PlsFormat format) +static std::pair pls_format_to_basetype(PlsFormat format) { switch (format) { @@ -234,17 +278,17 @@ static std::pair pls_format_to_basetype(PlsFormat f case PlsRGB10A2: case PlsRGBA8: case PlsRG16: - return std::make_pair(spv::OpTypeFloat, SPIRType::Float); + return std::make_pair(OpTypeFloat, SPIRType::Float); case PlsRGBA8I: case PlsRG16I: - return std::make_pair(spv::OpTypeInt, SPIRType::Int); + return std::make_pair(OpTypeInt, SPIRType::Int); case PlsRGB10A2UI: case PlsRGBA8UI: case PlsRG16UI: case PlsR32UI: - return std::make_pair(spv::OpTypeInt, SPIRType::UInt); + return std::make_pair(OpTypeInt, SPIRType::UInt); } } @@ -324,6 +368,7 @@ void CompilerGLSL::reset(uint32_t iteration_count) expression_usage_counts.clear(); forwarded_temporaries.clear(); suppressed_usage_tracking.clear(); + buffer_pointer_variables.clear(); // Ensure that we declare phi-variable copies even if the original declaration isn't deferred flushed_phi_variables.clear(); @@ -338,6 +383,7 @@ void CompilerGLSL::reset(uint32_t iteration_count) }); ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { var.dependees.clear(); }); + ir.for_each_typed_id([&](uint32_t, SPIRBlock &block) { block.rearm_dominated_variables.clear(); }); ir.reset_all_of_type(); ir.reset_all_of_type(); @@ -545,7 +591,7 @@ void CompilerGLSL::find_static_extensions() if (options.separate_shader_objects && !options.es && options.version < 410) require_extension_internal("GL_ARB_separate_shader_objects"); - if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64EXT) + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) { if (!options.vulkan_semantics) SPIRV_CROSS_THROW("GL_EXT_buffer_reference is only supported in Vulkan GLSL."); @@ -557,7 +603,7 @@ void CompilerGLSL::find_static_extensions() } else if (ir.addressing_model != AddressingModelLogical) { - SPIRV_CROSS_THROW("Only Logical and PhysicalStorageBuffer64EXT addressing models are supported."); + SPIRV_CROSS_THROW("Only Logical and PhysicalStorageBuffer64 addressing models are supported."); } // Check for nonuniform qualifier and passthrough. @@ -566,7 +612,7 @@ void CompilerGLSL::find_static_extensions() { switch (cap) { - case CapabilityShaderNonUniformEXT: + case CapabilityShaderNonUniform: if (!options.vulkan_semantics) require_extension_internal("GL_NV_gpu_shader5"); else @@ -610,6 +656,20 @@ void CompilerGLSL::find_static_extensions() ray_tracing_is_khr = true; break; + case CapabilityRayQueryPositionFetchKHR: + if (options.es || options.version < 460 || !options.vulkan_semantics) + SPIRV_CROSS_THROW("RayQuery Position Fetch requires Vulkan GLSL 460."); + require_extension_internal("GL_EXT_ray_tracing_position_fetch"); + ray_tracing_is_khr = true; + break; + + case CapabilityRayTracingPositionFetchKHR: + if (options.es || options.version < 460 || !options.vulkan_semantics) + SPIRV_CROSS_THROW("Ray Tracing Position Fetch requires Vulkan GLSL 460."); + require_extension_internal("GL_EXT_ray_tracing_position_fetch"); + ray_tracing_is_khr = true; + break; + case CapabilityRayTraversalPrimitiveCullingKHR: if (options.es || options.version < 460 || !options.vulkan_semantics) SPIRV_CROSS_THROW("RayQuery requires Vulkan GLSL 460."); @@ -617,6 +677,28 @@ void CompilerGLSL::find_static_extensions() ray_tracing_is_khr = true; break; + case CapabilityRayTracingClusterAccelerationStructureNV: + if (options.es || options.version < 460 || !options.vulkan_semantics) + SPIRV_CROSS_THROW("Cluster AS requires Vulkan GLSL 460."); + require_extension_internal("GL_NV_cluster_acceleration_structure"); + ray_tracing_is_khr = true; + break; + + case CapabilityTensorsARM: + if (options.es || options.version < 460 || !options.vulkan_semantics) + SPIRV_CROSS_THROW("Tensor requires Vulkan GLSL 460."); + require_extension_internal("GL_ARM_tensors"); + break; + + case CapabilityDescriptorHeapEXT: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("DescriptorHeapEXT requires Vulkan semantics."); + require_extension_internal("GL_EXT_descriptor_heap"); + require_extension_internal("GL_EXT_nonuniform_qualifier"); + // We lose information about writeonly/readonly in SPIR-V. Just pre-empt this to avoid complicating code later. + require_extension_internal("GL_EXT_shader_image_load_formatted"); + break; + default: break; } @@ -631,6 +713,12 @@ void CompilerGLSL::find_static_extensions() require_extension_internal("GL_OVR_multiview2"); } + if (execution.flags.get(ExecutionModeQuadDerivativesKHR) || + (execution.flags.get(ExecutionModeRequireFullQuadsKHR) && get_execution_model() == ExecutionModelFragment)) + { + require_extension_internal("GL_EXT_shader_quad_control"); + } + // KHR one is likely to get promoted at some point, so if we don't see an explicit SPIR-V extension, assume KHR. for (auto &ext : ir.declared_extensions) if (ext == "SPV_NV_fragment_shader_barycentric") @@ -639,7 +727,8 @@ void CompilerGLSL::find_static_extensions() void CompilerGLSL::require_polyfill(Polyfill polyfill, bool relaxed) { - uint32_t &polyfills = (relaxed && options.es) ? required_polyfills_relaxed : required_polyfills; + uint32_t &polyfills = (relaxed && (options.es || options.vulkan_semantics)) ? + required_polyfills_relaxed : required_polyfills; if ((polyfills & polyfill) == 0) { @@ -662,6 +751,46 @@ void CompilerGLSL::ray_tracing_khr_fixup_locations() }); } +std::string CompilerGLSL::integer_dot_product_entry_point(const IntegerDotProduct &idot) +{ + std::string expr = "spv"; + + switch (idot.op) + { + case OpSDot: expr += "SDot"; break; + case OpUDot: expr += "UDot"; break; + case OpSUDot: expr += "SUDot"; break; + case OpSDotAccSat: expr += "SDotAccSat"; break; + case OpUDotAccSat: expr += "UDotAccSat"; break; + case OpSUDotAccSat: expr += "SUDotAccSat"; break; + default: SPIRV_CROSS_THROW("Invalid integer dot product opcode."); + } + + expr += "_" + type_to_glsl(get(idot.result_type)); + for (auto &arg : idot.argument_type) + expr += "_" + type_to_glsl(get(arg)); + + return expr; +} + +void CompilerGLSL::add_integer_dot_product_polyfill(const IntegerDotProduct &idot) +{ + for (auto &impl : integer_dot_products_polyfills) + { + if (impl.result_type == idot.result_type && + impl.argument_type[0] == idot.argument_type[0] && + impl.argument_type[1] == idot.argument_type[1] && + impl.op == idot.op) + { + return; + } + } + + require_extension_internal("GL_EXT_spirv_intrinsics"); + integer_dot_products_polyfills.push_back(idot); + force_recompile(); +} + string CompilerGLSL::compile() { ir.fixup_reserved_names(); @@ -680,6 +809,8 @@ string CompilerGLSL::compile() backend.requires_relaxed_precision_analysis = options.es || options.vulkan_semantics; backend.support_precise_qualifier = (!options.es && options.version >= 400) || (options.es && options.version >= 320); + backend.constant_null_initializer = "{ }"; + backend.requires_matching_array_initializer = true; if (is_legacy_es()) backend.support_case_fallthrough = false; @@ -699,9 +830,20 @@ string CompilerGLSL::compile() // Shaders might cast unrelated data to pointers of non-block types. // Find all such instances and make sure we can cast the pointers to a synthesized block type. - if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64EXT) + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) analyze_non_block_pointer_types(); + if (std::find(ir.declared_capabilities.begin(), ir.declared_capabilities.end(), + CapabilityDescriptorHeapEXT) != ir.declared_capabilities.end()) + { + // Need to figure out all the aliased types that view the heap. + // In GLSL, each unique type must be declared with layout(descriptor_heap) type-decl spvSomeIdentResourceHeap[]; + // During untyped access chain traversal, we prefix the name to match the aliases. + // HLSL has more direct native support and will not need these, but we still need to call this function + // to verify that descriptor strides make sense. + analyze_descriptor_heap_types(); + } + uint32_t pass_count = 0; do { @@ -715,19 +857,32 @@ string CompilerGLSL::compile() if (required_polyfills != 0) emit_polyfills(required_polyfills, false); - if (options.es && required_polyfills_relaxed != 0) + if ((options.es || options.vulkan_semantics) && required_polyfills_relaxed != 0) emit_polyfills(required_polyfills_relaxed, true); + emit_polyfills_integer_dot_product(); - emit_function(get(ir.default_entry_point), Bitset()); + if (ir.is_library_module) + { + // Emit each exported function as a normal free function. + // emit_function recursively emits callees, so internal helpers + // are picked up too. + for (auto export_id : ir.library_exported_functions) + emit_function(get(export_id), Bitset()); + } + else + emit_function(get(ir.default_entry_point), Bitset()); pass_count++; } while (is_forcing_recompilation()); // Implement the interlocked wrapper function at the end. // The body was implemented in lieu of main(). - if (interlocked_is_complex) + if (interlocked_is_complex && !ir.is_library_module) { - statement("void main()"); + if (options.use_entry_point_name) + statement("void ", get_entry_point().name, "()"); + else + statement("void main()"); begin_scope(); statement("// Interlocks were used in a way not compatible with GLSL, this is very slow."); statement("SPIRV_Cross_beginInvocationInterlock();"); @@ -736,8 +891,10 @@ string CompilerGLSL::compile() end_scope(); } - // Entry point in GLSL is always main(). - get_entry_point().name = "main"; + // Entry point in GLSL is always main(). Skip the rename for library + // modules; their exports keep their declared names. + if (!options.use_entry_point_name && !ir.is_library_module) + get_entry_point().name = "main"; return buffer.str(); } @@ -809,6 +966,16 @@ void CompilerGLSL::request_subgroup_feature(ShaderSubgroupSupportHelper::Feature void CompilerGLSL::emit_header() { auto &execution = get_entry_point(); + + // Library modules have no entry point. The emitted GLSL is meant to be #include'd or appended + // rather than compiled standalone, so the version and extension directives that follow are + // wrapped in `#ifdef SPIRV_CROSS_LIBRARY_HEADER ... #endif`. By default they are skipped (the + // consuming translation unit provides its own preamble); a caller that wants to compile the + // library standalone defines SPIRV_CROSS_LIBRARY_HEADER to opt in. The stage-specific layout + // block at the end of this function is skipped entirely in library mode. + if (ir.is_library_module) + statement("#ifdef SPIRV_CROSS_LIBRARY_HEADER"); + statement("#version ", options.version, options.es && options.version > 100 ? " es" : ""); if (!options.es && options.version < 420) @@ -1018,6 +1185,13 @@ void CompilerGLSL::emit_header() for (auto &header : header_lines) statement(header); + if (ir.is_library_module) + { + statement("#endif"); + statement(""); + return; + } + SmallVector inputs; SmallVector outputs; @@ -1190,6 +1364,9 @@ void CompilerGLSL::emit_header() else if (!options.es && execution.flags.get(ExecutionModeDepthLess)) statement("layout(depth_less) out float gl_FragDepth;"); + if (execution.flags.get(ExecutionModeRequireFullQuadsKHR)) + statement("layout(full_quads) in;"); + break; default: @@ -1200,6 +1377,9 @@ void CompilerGLSL::emit_header() if (cap == CapabilityRayTraversalPrimitiveCullingKHR) statement("layout(primitive_culling);"); + if (execution.flags.get(ExecutionModeQuadDerivativesKHR)) + statement("layout(quad_derivatives) in;"); + if (!inputs.empty()) statement("layout(", merge(inputs), ") in;"); if (!outputs.empty()) @@ -1249,9 +1429,6 @@ void CompilerGLSL::emit_struct(SPIRType &type) emitted = true; } - if (has_extended_decoration(type.self, SPIRVCrossDecorationPaddingTarget)) - emit_struct_padding_target(type); - end_scope_decl(); if (emitted) @@ -1404,7 +1581,7 @@ string CompilerGLSL::layout_for_member(const SPIRType &type, uint32_t index) return res; } -const char *CompilerGLSL::format_to_glsl(spv::ImageFormat format) +const char *CompilerGLSL::format_to_glsl(ImageFormat format) { if (options.es && is_desktop_only_format(format)) SPIRV_CROSS_THROW("Attempting to use image format not supported in ES profile."); @@ -1514,9 +1691,12 @@ uint32_t CompilerGLSL::type_to_packed_base_size(const SPIRType &type, BufferPack case SPIRType::Half: case SPIRType::Short: case SPIRType::UShort: + case SPIRType::BFloat16: return 2; case SPIRType::SByte: case SPIRType::UByte: + case SPIRType::FloatE4M3: + case SPIRType::FloatE5M2: return 1; default: @@ -1527,14 +1707,14 @@ uint32_t CompilerGLSL::type_to_packed_base_size(const SPIRType &type, BufferPack uint32_t CompilerGLSL::type_to_packed_alignment(const SPIRType &type, const Bitset &flags, BufferPackingStandard packing) { - // If using PhysicalStorageBufferEXT storage class, this is a pointer, + // If using PhysicalStorageBuffer storage class, this is a pointer, // and is 64-bit. if (is_physical_pointer(type)) { if (!type.pointer) - SPIRV_CROSS_THROW("Types in PhysicalStorageBufferEXT must be pointers."); + SPIRV_CROSS_THROW("Types in PhysicalStorageBuffer must be pointers."); - if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64EXT) + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) { if (packing_is_vec4_padded(packing) && type_is_array_of_pointers(type)) return 16; @@ -1542,7 +1722,7 @@ uint32_t CompilerGLSL::type_to_packed_alignment(const SPIRType &type, const Bits return 8; } else - SPIRV_CROSS_THROW("AddressingModelPhysicalStorageBuffer64EXT must be used for PhysicalStorageBufferEXT."); + SPIRV_CROSS_THROW("AddressingModelPhysicalStorageBuffer64 must be used for PhysicalStorageBuffer."); } else if (is_array(type)) { @@ -1650,26 +1830,48 @@ uint32_t CompilerGLSL::type_to_packed_array_stride(const SPIRType &type, const B uint32_t CompilerGLSL::type_to_packed_size(const SPIRType &type, const Bitset &flags, BufferPackingStandard packing) { - // If using PhysicalStorageBufferEXT storage class, this is a pointer, + // If using PhysicalStorageBuffer storage class, this is a pointer, // and is 64-bit. if (is_physical_pointer(type)) { if (!type.pointer) - SPIRV_CROSS_THROW("Types in PhysicalStorageBufferEXT must be pointers."); + SPIRV_CROSS_THROW("Types in PhysicalStorageBuffer must be pointers."); - if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64EXT) + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) return 8; else - SPIRV_CROSS_THROW("AddressingModelPhysicalStorageBuffer64EXT must be used for PhysicalStorageBufferEXT."); + SPIRV_CROSS_THROW("AddressingModelPhysicalStorageBuffer64 must be used for PhysicalStorageBuffer."); } else if (is_array(type)) { uint32_t packed_size = to_array_size_literal(type) * type_to_packed_array_stride(type, flags, packing); - // For arrays of vectors and matrices in HLSL, the last element has a size which depends on its vector size, - // so that it is possible to pack other vectors into the last element. - if (packing_is_hlsl(packing) && type.basetype != SPIRType::Struct) - packed_size -= (4 - type.vecsize) * (type.width / 8); + if (packing_is_hlsl(packing)) + { + // For arrays of vectors and matrices in HLSL, the last element has a size which depends on its vector size, + // so that it is possible to pack other vectors into the last element. + if (type.basetype != SPIRType::Struct) + { + if (flags.get(DecorationRowMajor) && type.columns > 1) + packed_size -= (4 - type.columns) * (type.width / 8); + else + packed_size -= (4 - type.vecsize) * (type.width / 8); + } + else + { + const auto *base_type = &type; + while (is_array(*base_type)) + { + auto &new_type = get(base_type->parent_type); + if (!is_array(new_type)) + break; + base_type = &new_type; + } + + packed_size -= type_to_packed_array_stride(*base_type, flags, packing); + packed_size += type_to_packed_size(get(base_type->parent_type), flags, packing); + } + } return packed_size; } @@ -1688,15 +1890,27 @@ uint32_t CompilerGLSL::type_to_packed_size(const SPIRType &type, const Bitset &f uint32_t packed_alignment = type_to_packed_alignment(member_type, member_flags, packing); uint32_t alignment = max(packed_alignment, pad_alignment); - // The next member following a struct member is aligned to the base alignment of the struct that came before. - // GL 4.5 spec, 7.6.2.2. - if (member_type.basetype == SPIRType::Struct) - pad_alignment = packed_alignment; + uint32_t element_size = type_to_packed_size(member_type, member_flags, packing); + pad_alignment = 1; + + if (packing_is_hlsl(packing)) + { + // HLSL is primarily a "cannot-straddle-vec4" language. + uint32_t begin_word = size / 16; + uint32_t end_word = (size + element_size - 1) / 16; + if (begin_word != end_word) + alignment = max(alignment, 16u); + } else - pad_alignment = 1; + { + // The next member following a struct member is aligned to the base alignment of the struct that came before. + // GL 4.5 spec, 7.6.2.2. + if (member_type.basetype == SPIRType::Struct) + pad_alignment = packed_alignment; + } size = (size + alignment - 1) & ~(alignment - 1); - size += type_to_packed_size(member_type, member_flags, packing); + size += element_size; } } else @@ -1714,9 +1928,7 @@ uint32_t CompilerGLSL::type_to_packed_size(const SPIRType &type, const Bitset &f if (flags.get(DecorationColMajor) && type.columns > 1) { - if (packing_is_vec4_padded(packing)) - size = type.columns * 4 * base_alignment; - else if (type.vecsize == 3) + if (packing_is_vec4_padded(packing) || type.vecsize == 3) size = type.columns * 4 * base_alignment; else size = type.columns * type.vecsize * base_alignment; @@ -1724,9 +1936,7 @@ uint32_t CompilerGLSL::type_to_packed_size(const SPIRType &type, const Bitset &f if (flags.get(DecorationRowMajor) && type.vecsize > 1) { - if (packing_is_vec4_padded(packing)) - size = type.vecsize * 4 * base_alignment; - else if (type.columns == 3) + if (packing_is_vec4_padded(packing) || type.columns == 3) size = type.vecsize * 4 * base_alignment; else size = type.vecsize * type.columns * base_alignment; @@ -1735,7 +1945,12 @@ uint32_t CompilerGLSL::type_to_packed_size(const SPIRType &type, const Bitset &f // For matrices in HLSL, the last element has a size which depends on its vector size, // so that it is possible to pack other vectors into the last element. if (packing_is_hlsl(packing) && type.columns > 1) - size -= (4 - type.vecsize) * (type.width / 8); + { + if (flags.get(DecorationRowMajor)) + size -= (4 - type.columns) * (type.width / 8); + else + size -= (4 - type.vecsize) * (type.width / 8); + } } } @@ -1768,7 +1983,9 @@ bool CompilerGLSL::buffer_is_packing_standard(const SPIRType &type, BufferPackin for (uint32_t i = 0; i < type.member_types.size(); i++) { auto &memb_type = get(type.member_types[i]); - auto member_flags = ir.meta[type.self].members[i].decoration_flags; + + auto *type_meta = ir.find_meta(type.self); + auto member_flags = type_meta ? type_meta->members[i].decoration_flags : Bitset{}; // Verify alignment rules. uint32_t packed_alignment = type_to_packed_alignment(memb_type, member_flags, packing); @@ -1824,7 +2041,7 @@ bool CompilerGLSL::buffer_is_packing_standard(const SPIRType &type, BufferPackin // The next member following a struct member is aligned to the base alignment of the struct that came before. // GL 4.5 spec, 7.6.2.2. - if (memb_type.basetype == SPIRType::Struct && !memb_type.pointer) + if (!packing_is_hlsl(packing) && memb_type.basetype == SPIRType::Struct && !memb_type.pointer) pad_alignment = packed_alignment; else pad_alignment = 1; @@ -1852,13 +2069,16 @@ bool CompilerGLSL::buffer_is_packing_standard(const SPIRType &type, BufferPackin } // Verify array stride rules. - if (is_array(memb_type) && - type_to_packed_array_stride(memb_type, member_flags, packing) != - type_struct_member_array_stride(type, i)) + if (is_array(memb_type)) { - if (failed_validation_index) - *failed_validation_index = i; - return false; + auto packed_array_stride = type_to_packed_array_stride(memb_type, member_flags, packing); + auto member_array_stride = type_struct_member_array_stride(type, i); + if (packed_array_stride != member_array_stride) + { + if (failed_validation_index) + *failed_validation_index = i; + return false; + } } // Verify that sub-structs also follow packing rules. @@ -2145,11 +2365,11 @@ string CompilerGLSL::layout_for_variable(const SPIRVariable &var) // If SPIR-V does not comply with either layout, we cannot really work around it. if (can_use_buffer_blocks && (ubo_block || emulated_ubo)) { - attr.push_back(buffer_to_packing_standard(type, false)); + attr.push_back(buffer_to_packing_standard(type, false, true)); } else if (can_use_buffer_blocks && (push_constant_block || ssbo_block)) { - attr.push_back(buffer_to_packing_standard(type, true)); + attr.push_back(buffer_to_packing_standard(type, true, true)); } // For images, the type itself adds a layout qualifer. @@ -2170,7 +2390,9 @@ string CompilerGLSL::layout_for_variable(const SPIRVariable &var) return res; } -string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, bool support_std430_without_scalar_layout) +string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, + bool support_std430_without_scalar_layout, + bool support_enhanced_layouts) { if (support_std430_without_scalar_layout && buffer_is_packing_standard(type, BufferPackingStd430)) return "std430"; @@ -2182,6 +2404,7 @@ string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, bool suppo return "scalar"; } else if (support_std430_without_scalar_layout && + support_enhanced_layouts && buffer_is_packing_standard(type, BufferPackingStd430EnhancedLayout)) { if (options.es && !options.vulkan_semantics) @@ -2193,7 +2416,8 @@ string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, bool suppo set_extended_decoration(type.self, SPIRVCrossDecorationExplicitOffset); return "std430"; } - else if (buffer_is_packing_standard(type, BufferPackingStd140EnhancedLayout)) + else if (support_enhanced_layouts && + buffer_is_packing_standard(type, BufferPackingStd140EnhancedLayout)) { // Fallback time. We might be able to use the ARB_enhanced_layouts to deal with this difference, // however, we can only use layout(offset) on the block itself, not any substructs, so the substructs better be the appropriate layout. @@ -2207,7 +2431,9 @@ string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, bool suppo set_extended_decoration(type.self, SPIRVCrossDecorationExplicitOffset); return "std140"; } - else if (options.vulkan_semantics && buffer_is_packing_standard(type, BufferPackingScalarEnhancedLayout)) + else if (options.vulkan_semantics && + support_enhanced_layouts && + buffer_is_packing_standard(type, BufferPackingScalarEnhancedLayout)) { set_extended_decoration(type.self, SPIRVCrossDecorationExplicitOffset); require_extension_internal("GL_EXT_scalar_block_layout"); @@ -2221,6 +2447,7 @@ string CompilerGLSL::buffer_to_packing_standard(const SPIRType &type, bool suppo return "std430"; } else if (!support_std430_without_scalar_layout && options.vulkan_semantics && + support_enhanced_layouts && buffer_is_packing_standard(type, BufferPackingStd430EnhancedLayout)) { // UBOs can support std430 with GL_EXT_scalar_block_layout. @@ -2242,7 +2469,7 @@ void CompilerGLSL::emit_push_constant_block(const SPIRVariable &var) else if (options.vulkan_semantics) emit_push_constant_block_vulkan(var); else if (options.emit_push_constant_as_uniform_buffer) - emit_buffer_block_native(var); + emit_buffer_block_native(&var, nullptr); else emit_push_constant_block_glsl(var); } @@ -2291,7 +2518,7 @@ void CompilerGLSL::emit_buffer_block(const SPIRVariable &var) (ubo_block && options.emit_uniform_buffer_as_plain_uniforms)) emit_buffer_block_legacy(var); else - emit_buffer_block_native(var); + emit_buffer_block_native(&var, nullptr); } void CompilerGLSL::emit_buffer_block_legacy(const SPIRVariable &var) @@ -2319,7 +2546,7 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de auto &type = get(type_id); string buffer_name; - if (forward_declaration) + if (forward_declaration && is_physical_pointer_to_buffer_block(type)) { // Block names should never alias, but from HLSL input they kind of can because block types are reused for UAVs ... // Allow aliased name since we might be declaring the block twice. Once with buffer reference (forward declared) and one proper declaration. @@ -2352,10 +2579,10 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de // Ensure we emit the correct name when emitting non-forward pointer type. ir.meta[type.self].decoration.alias = buffer_name; } - else if (type.basetype != SPIRType::Struct) - buffer_name = type_to_glsl(type); else - buffer_name = to_name(type.self, false); + { + buffer_name = type_to_glsl(type); + } if (!forward_declaration) { @@ -2364,13 +2591,13 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de if (itr != physical_storage_type_to_alignment.end()) alignment = itr->second.alignment; - if (type.basetype == SPIRType::Struct) + if (is_physical_pointer_to_buffer_block(type)) { SmallVector attributes; attributes.push_back("buffer_reference"); if (alignment) attributes.push_back(join("buffer_reference_align = ", alignment)); - attributes.push_back(buffer_to_packing_standard(type, true)); + attributes.push_back(buffer_to_packing_standard(type, true, true)); auto flags = ir.get_buffer_block_type_flags(type); string decorations; @@ -2385,14 +2612,32 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de statement("layout(", merge(attributes), ")", decorations, " buffer ", buffer_name); } - else if (alignment) - statement("layout(buffer_reference, buffer_reference_align = ", alignment, ") buffer ", buffer_name); else - statement("layout(buffer_reference) buffer ", buffer_name); + { + string packing_standard; + if (type.basetype == SPIRType::Struct) + { + // The non-block type is embedded in a block, so we cannot use enhanced layouts :( + packing_standard = buffer_to_packing_standard(type, true, false) + ", "; + } + else if (is_array(get_pointee_type(type))) + { + SPIRType wrap_type{OpTypeStruct}; + wrap_type.self = ir.increase_bound_by(1); + wrap_type.member_types.push_back(get_pointee_type_id(type_id)); + ir.set_member_decoration(wrap_type.self, 0, DecorationOffset, 0); + packing_standard = buffer_to_packing_standard(wrap_type, true, false) + ", "; + } + + if (alignment) + statement("layout(", packing_standard, "buffer_reference, buffer_reference_align = ", alignment, ") buffer ", buffer_name); + else + statement("layout(", packing_standard, "buffer_reference) buffer ", buffer_name); + } begin_scope(); - if (type.basetype == SPIRType::Struct) + if (is_physical_pointer_to_buffer_block(type)) { type.member_name_cache.clear(); @@ -2407,7 +2652,7 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de else { auto &pointee_type = get_pointee_type(type); - statement(type_to_glsl(pointee_type), " value", type_to_array_glsl(pointee_type), ";"); + statement(type_to_glsl(pointee_type), " value", type_to_array_glsl(pointee_type, 0), ";"); } end_scope_decl(); @@ -2419,30 +2664,98 @@ void CompilerGLSL::emit_buffer_reference_block(uint32_t type_id, bool forward_de } } -void CompilerGLSL::emit_buffer_block_native(const SPIRVariable &var) +std::string CompilerGLSL::heap_meta_to_prefix(const DescriptorHeapMeta &meta) { - auto &type = get(var.basetype); + std::string prefix; + + if (meta.nonreadable) + prefix += "NoRead"; + if (meta.nonwritable) + prefix += "NoWrite"; + if (meta.coherent) + prefix += "Coherent"; + if (meta.is_volatile) + prefix += "Volatile"; + if (meta.is_restrict) + prefix += "Restrict"; + + return prefix; +} + +std::string CompilerGLSL::to_buffer_pointer_name_prefix(uint32_t ptr_id) const +{ + auto itr = std::find_if(descriptor_heap_types.begin(), descriptor_heap_types.end(), + [&](const DescriptorHeapMeta &meta) { return meta.buffer_pointer_id == ptr_id; }); + + assert(itr != descriptor_heap_types.end()); + + auto name = to_name(itr->type); + + // The same block type can be instantiated with different read-write decorations. + name += heap_meta_to_prefix(*itr); + + // Disambiguate since we can create multiple buffer pointers with same types. + name += to_name(itr->buffer_pointer_id); + + return join("spv", name); +} + +void CompilerGLSL::emit_buffer_block_native(const SPIRVariable *var, const DescriptorHeapMeta *heap_meta) +{ + assert(var || heap_meta); + + SPIRType *type; + if (var) + type = &get(var->basetype); + else + type = &get(heap_meta->type); + + Bitset flags = var ? ir.get_buffer_block_flags(*var) : ir.get_buffer_block_type_flags(*type); + auto storage = var ? var->storage : heap_meta->storage; + + if (heap_meta) + { + if (heap_meta->nonreadable) + flags.set(DecorationNonReadable); + if (heap_meta->nonwritable) + flags.set(DecorationNonWritable); + if (heap_meta->coherent) + flags.set(DecorationCoherent); + if (heap_meta->is_volatile) + flags.set(DecorationVolatile); + if (heap_meta->is_restrict) + flags.set(DecorationRestrict); + } + + bool ssbo = storage == StorageClassStorageBuffer || storage == StorageClassShaderRecordBufferKHR || + has_decoration(type->self, DecorationBufferBlock); - Bitset flags = ir.get_buffer_block_flags(var); - bool ssbo = var.storage == StorageClassStorageBuffer || var.storage == StorageClassShaderRecordBufferKHR || - ir.meta[type.self].decoration.decoration_flags.get(DecorationBufferBlock); bool is_restrict = ssbo && flags.get(DecorationRestrict); bool is_writeonly = ssbo && flags.get(DecorationNonReadable); bool is_readonly = ssbo && flags.get(DecorationNonWritable); bool is_coherent = ssbo && flags.get(DecorationCoherent); // Block names should never alias, but from HLSL input they kind of can because block types are reused for UAVs ... - auto buffer_name = to_name(type.self, false); + auto buffer_name = to_name(type->self, false); + + if (heap_meta) + { + // The same block type can be instantiated with different read-write decorations. + buffer_name += heap_meta_to_prefix(*heap_meta); + } auto &block_namespace = ssbo ? block_ssbo_names : block_ubo_names; // Shaders never use the block by interface name, so we don't // have to track this other than updating name caches. // If we have a collision for any reason, just fallback immediately. - if (ir.meta[type.self].decoration.alias.empty() || block_namespace.find(buffer_name) != end(block_namespace) || - resource_names.find(buffer_name) != end(resource_names)) + if (var) { - buffer_name = get_block_fallback_name(var.self); + if (ir.meta[type->self].decoration.alias.empty() || block_namespace.find(buffer_name) != end(block_namespace) || + resource_names.find(buffer_name) != end(resource_names)) + { + buffer_name = get_block_fallback_name(var->self); + } } // Make sure we get something unique for both global name scope and block name scope. @@ -2453,40 +2766,68 @@ void CompilerGLSL::emit_buffer_block_native(const SPIRVariable &var) // This cannot conflict with anything else, so we're safe now. // We cannot reuse this fallback name in neither global scope (blocked by block_names) nor block name scope. if (buffer_name.empty()) - buffer_name = join("_", get(var.basetype).self, "_", var.self); + { + if (var) + buffer_name = join("_", get(var->basetype).self, "_", var->self); + else + buffer_name = join("_", type->self); + } block_names.insert(buffer_name); block_namespace.insert(buffer_name); // Save for post-reflection later. - declared_block_names[var.self] = buffer_name; + if (var) + declared_block_names[var->self] = buffer_name; - statement(layout_for_variable(var), is_coherent ? "coherent " : "", is_restrict ? "restrict " : "", + string layout; + + if (var) + { + layout = layout_for_variable(*var); + } + else + { + auto packing_standard = buffer_to_packing_standard(*type, ssbo, true); + layout = join("layout(", + to_descriptor_heap_layout(*type, ssbo ? StorageClassStorageBuffer : StorageClassUniform), + ", ", packing_standard, ") "); + } + + statement(layout, is_coherent ? "coherent " : "", is_restrict ? "restrict " : "", is_writeonly ? "writeonly " : "", is_readonly ? "readonly " : "", ssbo ? "buffer " : "uniform ", buffer_name); begin_scope(); - type.member_name_cache.clear(); + type->member_name_cache.clear(); uint32_t i = 0; - for (auto &member : type.member_types) + for (auto &member : type->member_types) { - add_member_name(type, i); - emit_struct_member(type, member, i); + add_member_name(*type, i); + emit_struct_member(*type, member, i); i++; } // Don't declare empty blocks in GLSL, this is not allowed. - if (type_is_empty(type) && !backend.supports_empty_struct) + if (type_is_empty(*type) && !backend.supports_empty_struct) statement("int empty_struct_member;"); // var.self can be used as a backup name for the block name, // so we need to make sure we don't disturb the name here on a recompile. // It will need to be reset if we have to recompile. - preserve_alias_on_reset(var.self); - add_resource_name(var.self); - end_scope_decl(to_name(var.self) + type_to_array_glsl(type)); + if (var) + { + preserve_alias_on_reset(var->self); + add_resource_name(var->self); + end_scope_decl(to_name(var->self) + type_to_array_glsl(*type, var->self)); + } + else + { + end_scope_decl(join(to_buffer_pointer_name_prefix(heap_meta->buffer_pointer_id), "ResourceHeap[]")); + } + statement(""); } @@ -2508,7 +2849,7 @@ void CompilerGLSL::emit_buffer_block_flattened(const SPIRVariable &var) SPIRV_CROSS_THROW("Basic types in a flattened UBO must be float, int or uint."); auto flags = ir.get_buffer_block_flags(var); - statement("uniform ", flags_to_qualifiers_glsl(tmp, flags), type_to_glsl(tmp), " ", buffer_name, "[", + statement("uniform ", flags_to_qualifiers_glsl(tmp, 0, flags), type_to_glsl(tmp), " ", buffer_name, "[", buffer_size, "];"); } else @@ -2541,7 +2882,7 @@ const char *CompilerGLSL::to_storage_qualifiers_glsl(const SPIRVariable &var) return var.storage == StorageClassInput ? "in " : "out "; } else if (var.storage == StorageClassUniformConstant || var.storage == StorageClassUniform || - var.storage == StorageClassPushConstant) + var.storage == StorageClassPushConstant || var.storage == StorageClassAtomicCounter) { return "uniform "; } @@ -2736,6 +3077,8 @@ void CompilerGLSL::emit_interface_block(const SPIRVariable &var) block_qualifier = "patch "; else if (has_decoration(var.self, DecorationPerPrimitiveEXT)) block_qualifier = "perprimitiveEXT "; + else if (has_decoration(var.self, DecorationPerVertexKHR)) + block_qualifier = "pervertexEXT "; else block_qualifier = ""; @@ -2753,7 +3096,7 @@ void CompilerGLSL::emit_interface_block(const SPIRVariable &var) } add_resource_name(var.self); - end_scope_decl(join(to_name(var.self), type_to_array_glsl(type))); + end_scope_decl(join(to_name(var.self), type_to_array_glsl(type, var.self))); statement(""); } } @@ -2811,7 +3154,7 @@ void CompilerGLSL::emit_uniform(const SPIRVariable &var) statement(layout_for_variable(var), variable_decl(var), ";"); } -string CompilerGLSL::constant_value_macro_name(uint32_t id) +string CompilerGLSL::constant_value_macro_name(uint32_t id) const { return join("SPIRV_CROSS_CONSTANT_ID_", id); } @@ -3594,6 +3937,79 @@ void CompilerGLSL::emit_resources() bool emitted = false; + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) + { + // Output buffer reference block forward declarations. + ir.for_each_typed_id([&](uint32_t id, SPIRType &type) + { + if (is_physical_pointer(type)) + { + bool emit_type = true; + if (!is_physical_pointer_to_buffer_block(type)) + { + // Only forward-declare if we intend to emit it in the non_block_pointer types. + // Otherwise, these are just "benign" pointer types that exist as a result of access chains. + emit_type = std::find(physical_storage_non_block_pointer_types.begin(), + physical_storage_non_block_pointer_types.end(), + id) != physical_storage_non_block_pointer_types.end(); + } + + if (emit_type) + { + emit_buffer_reference_block(id, true); + emitted = true; + } + } + }); + } + + if (emitted) + statement(""); + emitted = false; + + SmallVector spec_const_dependencies; + bool legacy_spec_constant_workgroup = execution.model == ExecutionModelGLCompute && !options.vulkan_semantics && + (execution.workgroup_size.constant != 0 || execution.flags.get( + ExecutionModeLocalSizeId)); + if (legacy_spec_constant_workgroup) + { + SpecializationConstant wg_x, wg_y, wg_z; + get_work_group_size_specialization_constants(wg_x, wg_y, wg_z); + + if (wg_x.id != ConstantID(0)) + spec_const_dependencies.push_back(wg_x.id); + if (wg_y.id != ConstantID(0)) + spec_const_dependencies.push_back(wg_y.id); + if (wg_z.id != ConstantID(0)) + spec_const_dependencies.push_back(wg_z.id); + } + + const auto notify_spec_constant = [&](ConstantID id) + { + if (legacy_spec_constant_workgroup) + { + auto itr = std::find(spec_const_dependencies.begin(), spec_const_dependencies.end(), id); + + if (itr == spec_const_dependencies.end()) + return; + + spec_const_dependencies.erase(itr); + if (spec_const_dependencies.empty()) + { + SpecializationConstant wg_x, wg_y, wg_z; + // We have declared all dependencies. We must delcare the workgroup size immediately + // as subsequent spec constant ops may depend on the declaration. + // Newer glslang does not allow gl_WorkGroupSize to be accessed before layout(local_size) in; + get_work_group_size_specialization_constants(wg_x, wg_y, wg_z); + SmallVector inputs; + build_workgroup_size(inputs, wg_x, wg_y, wg_z); + statement("layout(", merge(inputs), ") in;"); + statement(""); + legacy_spec_constant_workgroup = false; + } + } + }; + // If emitted Vulkan GLSL, // emit specialization constants as actual floats, // spec op expressions will redirect to the constant name. @@ -3624,11 +4040,15 @@ void CompilerGLSL::emit_resources() emit_constant(c); emitted = true; } + + if (c.specialization) + notify_spec_constant(ConstantID(c.self)); } else if (id.get_type() == TypeConstantOp) { emit_specialization_constant_op(id.get()); emitted = true; + notify_spec_constant(ConstantID(id.get_id())); } else if (id.get_type() == TypeType) { @@ -3663,11 +4083,11 @@ void CompilerGLSL::emit_resources() auto &type = this->get(undef.basetype); // OpUndef can be void for some reason ... if (type.basetype == SPIRType::Void) - return; + continue; // This will break. It is bogus and should not be legal. if (type_is_top_level_block(type)) - return; + continue; string initializer; if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) @@ -3683,53 +4103,18 @@ void CompilerGLSL::emit_resources() if (emitted) statement(""); - // If we needed to declare work group size late, check here. - // If the work group size depends on a specialization constant, we need to declare the layout() block - // after constants (and their macros) have been declared. - if (execution.model == ExecutionModelGLCompute && !options.vulkan_semantics && - (execution.workgroup_size.constant != 0 || execution.flags.get(ExecutionModeLocalSizeId))) - { - SpecializationConstant wg_x, wg_y, wg_z; - get_work_group_size_specialization_constants(wg_x, wg_y, wg_z); - - if ((wg_x.id != ConstantID(0)) || (wg_y.id != ConstantID(0)) || (wg_z.id != ConstantID(0))) - { - SmallVector inputs; - build_workgroup_size(inputs, wg_x, wg_y, wg_z); - statement("layout(", merge(inputs), ") in;"); - statement(""); - } - } - emitted = false; - if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64EXT) + if (ir.addressing_model == AddressingModelPhysicalStorageBuffer64) { - for (auto type : physical_storage_non_block_pointer_types) - { - emit_buffer_reference_block(type, false); - } - // Output buffer reference blocks. - // Do this in two stages, one with forward declaration, - // and one without. Buffer reference blocks can reference themselves - // to support things like linked lists. - ir.for_each_typed_id([&](uint32_t self, SPIRType &type) { - if (type.basetype == SPIRType::Struct && type.pointer && - type.pointer_depth == 1 && !type_is_array_of_pointers(type) && - type.storage == StorageClassPhysicalStorageBufferEXT) - { - emit_buffer_reference_block(self, true); - } - }); + // Buffer reference blocks can reference themselves to support things like linked lists. + for (auto type : physical_storage_non_block_pointer_types) + emit_buffer_reference_block(type, false); - ir.for_each_typed_id([&](uint32_t self, SPIRType &type) { - if (type.basetype == SPIRType::Struct && - type.pointer && type.pointer_depth == 1 && !type_is_array_of_pointers(type) && - type.storage == StorageClassPhysicalStorageBufferEXT) - { - emit_buffer_reference_block(self, false); - } + ir.for_each_typed_id([&](uint32_t id, SPIRType &type) { + if (is_physical_pointer_to_buffer_block(type)) + emit_buffer_reference_block(id, false); }); } @@ -3891,6 +4276,45 @@ void CompilerGLSL::emit_resources() } } + bool needs_hlsl_warning = false; + + for (const auto &heap_type : descriptor_heap_types) + { + auto &type = get(heap_type.type); + + if (heap_type.hlsl_style_stride) + needs_hlsl_warning = true; + + if (type.basetype == SPIRType::Image || type.basetype == SPIRType::AccelerationStructure) + { + string type_layout; + + // We lose NonWritable / NonReadable information, glslang issue, no good way to plumb it through either ... + if (type.basetype == SPIRType::Image && type.image.sampled == 2 && type.image.format != ImageFormatUnknown) + type_layout = join("layout(", to_descriptor_heap_layout(type), ", ", format_to_glsl(type.image.format), ") uniform "); + else + type_layout = join("layout(", to_descriptor_heap_layout(type), ") uniform "); + + statement(type_layout, variable_decl(type, join("spv", to_name(type.self), "ResourceHeap")), "[];"); + } + else if (type.basetype == SPIRType::Sampler) + { + statement("layout(", to_descriptor_heap_layout(type), ") uniform ", + variable_decl(type, join("spv", to_name(type.self), "SamplerHeap")), "[];"); + } + else + { + emit_buffer_block_native(nullptr, &heap_type); + } + } + + if (needs_hlsl_warning) + { + statement("// WARNING: HLSL style descriptor heap stride is assumed for one or more descriptors. Allowing for compatibility with HLSL shaders."); + statement("// This may be not strictly be compatible with GLSL if sizeof(buffer) != sizeof(image)."); + statement("// Application side can convert bindless indices accordingly to compensate or use explicit mapping API to configure strides outside SPIRV-Cross."); + } + if (emitted) statement(""); } @@ -3947,7 +4371,7 @@ void CompilerGLSL::emit_output_variable_initializer(const SPIRVariable &var) auto &c = get(var.initializer); for (uint32_t j = 0; j < array_size; j++) exprs.push_back(to_expression(get(c.subconstants[j]).subconstants[i])); - statement("const ", type_to_glsl(array_type), " ", lut_name, type_to_array_glsl(array_type), " = ", + statement("const ", type_to_glsl(array_type), " ", lut_name, type_to_array_glsl(array_type, 0), " = ", type_to_glsl_constructor(array_type), "(", merge(exprs, ", "), ");"); } @@ -3980,19 +4404,19 @@ void CompilerGLSL::emit_output_variable_initializer(const SPIRVariable &var) if (type_is_array && !is_control_point) { uint32_t indices[2] = { j, i }; - auto chain = access_chain_internal(var.self, indices, 2, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &meta); + auto chain = access_chain_internal(var.self, indices, 2, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &meta, nullptr); statement(chain, " = ", lut_name, "[", j, "];"); } else if (is_control_point) { uint32_t indices[2] = { invocation_id, member_index_id }; - auto chain = access_chain_internal(var.self, indices, 2, 0, &meta); + auto chain = access_chain_internal(var.self, indices, 2, 0, &meta, nullptr); statement(chain, " = ", lut_name, "[", builtin_to_glsl(BuiltInInvocationId, StorageClassInput), "];"); } else { auto chain = - access_chain_internal(var.self, &i, 1, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &meta); + access_chain_internal(var.self, &i, 1, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &meta, nullptr); statement(chain, " = ", to_expression(c.subconstants[i]), ";"); } @@ -4005,7 +4429,7 @@ void CompilerGLSL::emit_output_variable_initializer(const SPIRVariable &var) else if (is_control_point) { auto lut_name = join("_", var.self, "_init"); - statement("const ", type_to_glsl(type), " ", lut_name, type_to_array_glsl(type), + statement("const ", type_to_glsl(type), " ", lut_name, type_to_array_glsl(type, 0), " = ", to_expression(var.initializer), ";"); entry_func.fixup_hooks_in.push_back([&, lut_name]() { statement(to_expression(var.self), "[gl_InvocationID] = ", lut_name, "[gl_InvocationID];"); @@ -4030,7 +4454,7 @@ void CompilerGLSL::emit_output_variable_initializer(const SPIRVariable &var) { auto lut_name = join("_", var.self, "_init"); statement("const ", type_to_glsl(type), " ", lut_name, - type_to_array_glsl(type), " = ", to_expression(var.initializer), ";"); + type_to_array_glsl(type, var.self), " = ", to_expression(var.initializer), ";"); entry_func.fixup_hooks_in.push_back([&, lut_name, is_patch]() { if (is_patch) { @@ -4207,7 +4631,7 @@ void CompilerGLSL::emit_subgroup_arithmetic_workaround(const std::string &func, } } -void CompilerGLSL::emit_extension_workarounds(spv::ExecutionModel model) +void CompilerGLSL::emit_extension_workarounds(ExecutionModel model) { static const char *workaround_types[] = { "int", "ivec2", "ivec3", "ivec4", "uint", "uvec2", "uvec3", "uvec4", "float", "vec2", "vec3", "vec4", "double", "dvec2", "dvec3", "dvec4" }; @@ -4526,7 +4950,7 @@ void CompilerGLSL::emit_extension_workarounds(spv::ExecutionModel model) if (shader_subgroup_supporter.is_feature_requested(Supp::SubgroupMemBarrier)) { - if (model == spv::ExecutionModelGLCompute) + if (model == ExecutionModelGLCompute) { statement("#ifndef GL_KHR_shader_subgroup_basic"); statement("void subgroupMemoryBarrier() { groupMemoryBarrier(); }"); @@ -4612,7 +5036,7 @@ void CompilerGLSL::emit_extension_workarounds(spv::ExecutionModel model) } auto arithmetic_feature_helper = - [&](Supp::Feature feat, std::string func_name, spv::Op op, spv::GroupOperation group_op) + [&](Supp::Feature feat, std::string func_name, Op op, GroupOperation group_op) { if (shader_subgroup_supporter.is_feature_requested(feat)) { @@ -4685,6 +5109,37 @@ void CompilerGLSL::emit_extension_workarounds(spv::ExecutionModel model) } } +void CompilerGLSL::emit_polyfills_integer_dot_product() +{ + for (auto &op : integer_dot_products_polyfills) + { + string caps = join("[", CapabilityDotProduct); + auto &arg_type = get(op.argument_type[0]); + if (arg_type.basetype == SPIRType::SByte || arg_type.basetype == SPIRType::UByte) + caps += join(", ", CapabilityDotProductInput4x8Bit); + else if (arg_type.vecsize == 1) + caps += join(", ", CapabilityDotProductInput4x8BitPacked); + else + caps += join(", ", CapabilityDotProductInputAll); + caps += "]"; + + auto arg0 = type_to_glsl(get(op.argument_type[0])); + auto arg1 = type_to_glsl(get(op.argument_type[1])); + auto acc_arg = + (op.op == OpSDotAccSat || op.op == OpUDotAccSat || op.op == OpSUDotAccSat) + ? (", " + type_to_glsl(get(op.result_type))) : ""; + + bool packed_vector = get(op.argument_type[0]).vecsize == 1; + const char *packed_argument = packed_vector ? ", spirv_literal uint packedFormat" : ""; + + statement("spirv_instruction (extensions = [\"SPV_KHR_integer_dot_product\"], capabilities = ", + caps, ", id = ", op.op, ")"); + statement(type_to_glsl(get(op.result_type)), " ", integer_dot_product_entry_point(op), "(", + arg0, " arg0, ", arg1, " arg1", acc_arg, packed_argument, ");"); + statement(""); + } +} + void CompilerGLSL::emit_polyfills(uint32_t polyfills, bool relaxed) { const char *qual = ""; @@ -4812,19 +5267,126 @@ void CompilerGLSL::emit_polyfills(uint32_t polyfills, bool relaxed) end_scope(); statement(""); } + + if (!relaxed) + { + static const Polyfill polys[3][3] = { + { PolyfillNMin16, PolyfillNMin32, PolyfillNMin64 }, + { PolyfillNMax16, PolyfillNMax32, PolyfillNMax64 }, + { PolyfillNClamp16, PolyfillNClamp32, PolyfillNClamp64 }, + }; + + static const GLSLstd450 glsl_ops[] = { GLSLstd450NMin, GLSLstd450NMax, GLSLstd450NClamp }; + static const char *spv_ops[] = { "spvNMin", "spvNMax", "spvNClamp" }; + bool has_poly = false; + + for (uint32_t i = 0; i < 3; i++) + { + for (uint32_t j = 0; j < 3; j++) + { + if ((polyfills & polys[i][j]) == 0) + continue; + + const char *types[3][4] = { + { "float16_t", "f16vec2", "f16vec3", "f16vec4" }, + { "float", "vec2", "vec3", "vec4" }, + { "double", "dvec2", "dvec3", "dvec4" }, + }; + + for (uint32_t k = 0; k < 4; k++) + { + auto *type = types[j][k]; + + if (i < 2) + { + statement("spirv_instruction(set = \"GLSL.std.450\", id = ", glsl_ops[i], ") ", + type, " ", spv_ops[i], "(", type, ", ", type, ");"); + } + else + { + statement("spirv_instruction(set = \"GLSL.std.450\", id = ", glsl_ops[i], ") ", + type, " ", spv_ops[i], "(", type, ", ", type, ", ", type, ");"); + } + + has_poly = true; + } + } + } + + if (has_poly) + statement(""); + } + else + { + // Mediump intrinsics don't work correctly, so wrap the intrinsic in an outer shell that ensures mediump + // propagation. + + static const Polyfill polys[3][3] = { + { PolyfillNMin16, PolyfillNMin32, PolyfillNMin64 }, + { PolyfillNMax16, PolyfillNMax32, PolyfillNMax64 }, + { PolyfillNClamp16, PolyfillNClamp32, PolyfillNClamp64 }, + }; + + static const char *spv_ops[] = { "spvNMin", "spvNMax", "spvNClamp" }; + + for (uint32_t i = 0; i < 3; i++) + { + for (uint32_t j = 0; j < 3; j++) + { + if ((polyfills & polys[i][j]) == 0) + continue; + + const char *types[3][4] = { + { "float16_t", "f16vec2", "f16vec3", "f16vec4" }, + { "float", "vec2", "vec3", "vec4" }, + { "double", "dvec2", "dvec3", "dvec4" }, + }; + + for (uint32_t k = 0; k < 4; k++) + { + auto *type = types[j][k]; + + if (i < 2) + { + statement("mediump ", type, " ", spv_ops[i], "Relaxed(", + "mediump ", type, " a, mediump ", type, " b)"); + begin_scope(); + statement("mediump ", type, " res = ", spv_ops[i], "(a, b);"); + statement("return res;"); + end_scope(); + statement(""); + } + else + { + statement("mediump ", type, " ", spv_ops[i], "Relaxed(", + "mediump ", type, " a, mediump ", type, " b, mediump ", type, " c)"); + begin_scope(); + statement("mediump ", type, " res = ", spv_ops[i], "(a, b, c);"); + statement("return res;"); + end_scope(); + statement(""); + } + } + } + } + } } // Returns a string representation of the ID, usable as a function arg. // Default is to simply return the expression representation fo the arg ID. // Subclasses may override to modify the return value. -string CompilerGLSL::to_func_call_arg(const SPIRFunction::Parameter &, uint32_t id) +string CompilerGLSL::to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_t id) { + // BDA expects pointers through function interface. + if (!arg.alias_global_variable && is_physical_or_buffer_pointer(expression_type(id))) + return to_pointer_expression(id); + // Make sure that we use the name of the original variable, and not the parameter alias. uint32_t name_id = id; auto *var = maybe_get(id); if (var && var->basevariable) name_id = var->basevariable; - return to_expression(name_id); + return to_unpacked_expression(name_id); } void CompilerGLSL::force_temporary_and_recompile(uint32_t id) @@ -5011,11 +5573,8 @@ string CompilerGLSL::dereference_expression(const SPIRType &expr_type, const std return expr.substr(1); else if (backend.native_pointers) return join('*', expr); - else if (expr_type.storage == StorageClassPhysicalStorageBufferEXT && expr_type.basetype != SPIRType::Struct && - expr_type.pointer_depth == 1) - { + else if (is_physical_pointer(expr_type) && !is_physical_pointer_to_buffer_block(expr_type)) return join(enclose_expression(expr), ".value"); - } else return expr; } @@ -5082,7 +5641,8 @@ string CompilerGLSL::to_enclosed_unpacked_expression(uint32_t id, bool register_ string CompilerGLSL::to_dereferenced_expression(uint32_t id, bool register_expression_read) { auto &type = expression_type(id); - if (type.pointer && should_dereference(id)) + + if (is_pointer(type) && should_dereference(id)) return dereference_expression(type, to_enclosed_expression(id, register_expression_read)); else return to_expression(id, register_expression_read); @@ -5091,7 +5651,7 @@ string CompilerGLSL::to_dereferenced_expression(uint32_t id, bool register_expre string CompilerGLSL::to_pointer_expression(uint32_t id, bool register_expression_read) { auto &type = expression_type(id); - if (type.pointer && expression_is_lvalue(id) && !should_dereference(id)) + if (is_pointer(type) && expression_is_lvalue(id) && !should_dereference(id)) return address_of_expression(to_enclosed_expression(id, register_expression_read)); else return to_unpacked_expression(id, register_expression_read); @@ -5100,7 +5660,7 @@ string CompilerGLSL::to_pointer_expression(uint32_t id, bool register_expression string CompilerGLSL::to_enclosed_pointer_expression(uint32_t id, bool register_expression_read) { auto &type = expression_type(id); - if (type.pointer && expression_is_lvalue(id) && !should_dereference(id)) + if (is_pointer(type) && expression_is_lvalue(id) && !should_dereference(id)) return address_of_expression(to_enclosed_expression(id, register_expression_read)); else return to_enclosed_unpacked_expression(id, register_expression_read); @@ -5251,12 +5811,21 @@ string CompilerGLSL::to_non_uniform_aware_expression(uint32_t id) { string expr = to_expression(id); - if (has_decoration(id, DecorationNonUniform)) + if (is_descriptor_non_uniform(id)) convert_non_uniform_expression(expr, id); return expr; } +string CompilerGLSL::to_atomic_ptr_expression(uint32_t id) +{ + string expr = to_non_uniform_aware_expression(id); + // If we have naked pointer to POD, we need to dereference to get the proper ".value" resolve. + if (should_dereference(id)) + expr = dereference_expression(expression_type(id), expr); + return expr; +} + string CompilerGLSL::to_expression(uint32_t id, bool register_expression_read) { auto itr = invalid_expressions.find(id); @@ -5620,8 +6189,22 @@ string CompilerGLSL::constant_op_expression(const SPIRConstantOp &cop) case OpCompositeExtract: { - auto expr = access_chain_internal(cop.arguments[0], &cop.arguments[1], uint32_t(cop.arguments.size() - 1), - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr); + // Trivial vector extracts (of WorkGroupSize typically), + // punch through to the input spec constant if the composite is used as array size. + const auto *c = maybe_get(cop.arguments[0]); + + string expr; + if (c && cop.arguments.size() == 2 && c->is_used_as_array_length && + !backend.supports_spec_constant_array_size && + is_vector(get(c->constant_type))) + { + expr = to_expression(c->specialization_constant_id(0, cop.arguments[1])); + } + else + { + expr = access_chain_internal(cop.arguments[0], &cop.arguments[1], uint32_t(cop.arguments.size() - 1), + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr, nullptr); + } return expr; } @@ -5764,6 +6347,35 @@ string CompilerGLSL::constant_expression(const SPIRConstant &c, { return backend.null_pointer_literal; } + else if (c.is_null_array_specialized_length && backend.requires_matching_array_initializer) + { + require_extension_internal("GL_EXT_null_initializer"); + return backend.constant_null_initializer; + } + else if (c.replicated && type.op != OpTypeArray) + { + if (type.op == OpTypeMatrix) + { + uint32_t num_elements = type.columns; + // GLSL does not allow the replication constructor for matrices + // mat4(vec4(0.0)) needs to be manually expanded to mat4(vec4(0.0), vec4(0.0), vec4(0.0), vec4(0.0)); + std::string res; + res += type_to_glsl(type); + res += "("; + for (uint32_t i = 0; i < num_elements; i++) + { + res += to_expression(c.subconstants[0]); + if (i < num_elements - 1) + res += ", "; + } + res += ")"; + return res; + } + else + { + return join(type_to_glsl(type), "(", to_expression(c.subconstants[0]), ")"); + } + } else if (!c.subconstants.empty()) { // Handles Arrays and structures. @@ -5813,8 +6425,16 @@ string CompilerGLSL::constant_expression(const SPIRConstant &c, } uint32_t subconstant_index = 0; - for (auto &elem : c.subconstants) + size_t num_elements = c.subconstants.size(); + if (c.replicated) { + if (type.array.size() != 1) + SPIRV_CROSS_THROW("Multidimensional arrays not yet supported as replicated constans"); + num_elements = type.array[0]; + } + for (size_t i = 0; i < num_elements; i++) + { + auto &elem = c.subconstants[c.replicated ? 0 : i]; if (auto *op = maybe_get(elem)) { res += constant_op_expression(*op); @@ -5845,7 +6465,7 @@ string CompilerGLSL::constant_expression(const SPIRConstant &c, } } - if (&elem != &c.subconstants.back()) + if (i != num_elements - 1) res += ", "; subconstant_index++; @@ -5869,7 +6489,7 @@ string CompilerGLSL::constant_expression(const SPIRConstant &c, else return join(type_to_glsl(type), "(0)"); } - else if (c.columns() == 1) + else if (c.columns() == 1 && type.op != OpTypeCooperativeMatrixKHR) { auto res = constant_expression_vector(c, 0); @@ -5919,17 +6539,44 @@ string CompilerGLSL::constant_expression(const SPIRConstant &c, #pragma warning(disable : 4996) #endif +string CompilerGLSL::convert_floate4m3_to_string(const SPIRConstant &c, uint32_t col, uint32_t row) +{ + string res; + float float_value = c.scalar_floate4m3(col, row); + + // There is no infinity in e4m3. + if (std::isnan(float_value)) + { + SPIRType type { OpTypeFloat }; + type.basetype = SPIRType::Half; + type.vecsize = 1; + type.columns = 1; + res = join(type_to_glsl(type), "(0.0 / 0.0)"); + } + else + { + SPIRType type { OpTypeFloat }; + type.basetype = SPIRType::FloatE4M3; + type.vecsize = 1; + type.columns = 1; + res = join(type_to_glsl(type), "(", format_float(float_value), ")"); + } + + return res; +} + string CompilerGLSL::convert_half_to_string(const SPIRConstant &c, uint32_t col, uint32_t row) { string res; - float float_value = c.scalar_f16(col, row); + bool is_bfloat8 = get(c.constant_type).basetype == SPIRType::FloatE5M2; + float float_value = is_bfloat8 ? c.scalar_bf8(col, row) : c.scalar_f16(col, row); // There is no literal "hf" in GL_NV_gpu_shader5, so to avoid lots // of complicated workarounds, just value-cast to the half type always. if (std::isnan(float_value) || std::isinf(float_value)) { SPIRType type { OpTypeFloat }; - type.basetype = SPIRType::Half; + type.basetype = is_bfloat8 ? SPIRType::FloatE5M2 : SPIRType::Half; type.vecsize = 1; type.columns = 1; @@ -5945,7 +6592,7 @@ string CompilerGLSL::convert_half_to_string(const SPIRConstant &c, uint32_t col, else { SPIRType type { OpTypeFloat }; - type.basetype = SPIRType::Half; + type.basetype = is_bfloat8 ? SPIRType::FloatE5M2 : SPIRType::Half; type.vecsize = 1; type.columns = 1; res = join(type_to_glsl(type), "(", format_float(float_value), ")"); @@ -5957,7 +6604,9 @@ string CompilerGLSL::convert_half_to_string(const SPIRConstant &c, uint32_t col, string CompilerGLSL::convert_float_to_string(const SPIRConstant &c, uint32_t col, uint32_t row) { string res; - float float_value = c.scalar_f32(col, row); + + bool is_bfloat16 = get(c.constant_type).basetype == SPIRType::BFloat16; + float float_value = is_bfloat16 ? c.scalar_bf16(col, row) : c.scalar_f32(col, row); if (std::isnan(float_value) || std::isinf(float_value)) { @@ -6021,6 +6670,9 @@ string CompilerGLSL::convert_float_to_string(const SPIRConstant &c, uint32_t col res += "f"; } + if (is_bfloat16) + res = join("bfloat16_t(", res, ")"); + return res; } @@ -6177,6 +6829,29 @@ string CompilerGLSL::constant_expression_vector(const SPIRConstant &c, uint32_t switch (type.basetype) { + case SPIRType::FloatE4M3: + if (splat || swizzle_splat) + { + res += convert_floate4m3_to_string(c, vector, 0); + if (swizzle_splat) + res = remap_swizzle(get(c.constant_type), 1, res); + } + else + { + for (uint32_t i = 0; i < c.vector_size(); i++) + { + if (c.vector_size() > 1 && c.specialization_constant_id(vector, i) != 0) + res += to_expression(c.specialization_constant_id(vector, i)); + else + res += convert_floate4m3_to_string(c, vector, i); + + if (i + 1 < c.vector_size()) + res += ", "; + } + } + break; + + case SPIRType::FloatE5M2: case SPIRType::Half: if (splat || swizzle_splat) { @@ -6199,6 +6874,7 @@ string CompilerGLSL::constant_expression_vector(const SPIRConstant &c, uint32_t } break; + case SPIRType::BFloat16: case SPIRType::Float: if (splat || swizzle_splat) { @@ -6304,7 +6980,7 @@ string CompilerGLSL::constant_expression_vector(const SPIRConstant &c, uint32_t if (splat) { res += convert_to_string(c.scalar(vector, 0)); - if (is_legacy()) + if (is_legacy() && !has_extension("GL_EXT_gpu_shader4")) { // Fake unsigned constant literals with signed ones if possible. // Things like array sizes, etc, tend to be unsigned even though they could just as easily be signed. @@ -6323,7 +6999,7 @@ string CompilerGLSL::constant_expression_vector(const SPIRConstant &c, uint32_t else { res += convert_to_string(c.scalar(vector, i)); - if (is_legacy()) + if (is_legacy() && !has_extension("GL_EXT_gpu_shader4")) { // Fake unsigned constant literals with signed ones if possible. // Things like array sizes, etc, tend to be unsigned even though they could just as easily be signed. @@ -6540,10 +7216,20 @@ void CompilerGLSL::emit_uninitialized_temporary(uint32_t result_type, uint32_t r if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) initializer = join(" = ", to_zero_initialized_expression(result_type)); - statement(flags_to_qualifiers_glsl(type, flags), variable_decl(type, to_name(result_id)), initializer, ";"); + statement(flags_to_qualifiers_glsl(type, result_id, flags), variable_decl(type, to_name(result_id)), initializer, ";"); } } +bool CompilerGLSL::can_declare_inline_temporary(uint32_t id) const +{ + if (!block_temporary_hoisting && current_continue_block && !hoisted_temporaries.count(id)) + return false; + if (hoisted_temporaries.count(id)) + return false; + + return true; +} + string CompilerGLSL::declare_temporary(uint32_t result_type, uint32_t result_id) { auto &type = get(result_type); @@ -6575,7 +7261,7 @@ string CompilerGLSL::declare_temporary(uint32_t result_type, uint32_t result_id) // The result_id has not been made into an expression yet, so use flags interface. add_local_variable_name(result_id); auto &flags = get_decoration_bitset(result_id); - return join(flags_to_qualifiers_glsl(type, flags), variable_decl(type, to_name(result_id)), " = "); + return join(flags_to_qualifiers_glsl(type, result_id, flags), variable_decl(type, to_name(result_id)), " = "); } } @@ -6621,6 +7307,42 @@ SPIRExpression &CompilerGLSL::emit_op(uint32_t result_type, uint32_t result_id, } } +void CompilerGLSL::emit_transposed_op(uint32_t result_type, uint32_t result_id, const string &rhs, bool forwarding) +{ + if (forwarding && (forced_temporaries.find(result_id) == end(forced_temporaries))) + { + // Just forward it without temporary. + // If the forward is trivial, we do not force flushing to temporary for this expression. + forwarded_temporaries.insert(result_id); + auto &e = set(result_id, rhs, result_type, true); + e.need_transpose = true; + } + else if (can_declare_inline_temporary(result_id)) + { + // If expression isn't immutable, bind it to a temporary and make the new temporary immutable (they always are). + // Since the expression is transposed, we have to ensure the temporary is the transposed type. + + auto &transposed_type_id = extra_sub_expressions[result_id]; + if (!transposed_type_id) + { + auto dummy_type = get(result_type); + std::swap(dummy_type.columns, dummy_type.vecsize); + transposed_type_id = ir.increase_bound_by(1); + set(transposed_type_id, dummy_type); + } + + statement(declare_temporary(transposed_type_id, result_id), rhs, ";"); + auto &e = set(result_id, to_name(result_id), result_type, true); + e.need_transpose = true; + } + else + { + // If we cannot declare the temporary because it's already been hoisted, we don't have the + // chance to override the temporary type ourselves. Just transpose() the expression. + emit_op(result_type, result_id, join("transpose(", rhs, ")"), forwarding); + } +} + void CompilerGLSL::emit_unary_op(uint32_t result_type, uint32_t result_id, uint32_t op0, const char *op) { bool forward = should_forward(op0); @@ -6648,7 +7370,7 @@ void CompilerGLSL::emit_binary_op(uint32_t result_type, uint32_t result_id, uint { // Various FP arithmetic opcodes such as add, sub, mul will hit this. bool force_temporary_precise = backend.support_precise_qualifier && - has_decoration(result_id, DecorationNoContraction) && + has_legacy_nocontract(result_type, result_id) && type_is_floating_point(get(result_type)); bool forward = should_forward(op0) && should_forward(op1) && !force_temporary_precise; @@ -6854,9 +7576,12 @@ void CompilerGLSL::emit_atomic_func_op(uint32_t result_type, uint32_t result_id, require_extension_internal("GL_EXT_shader_atomic_float"); } + if (type.basetype == SPIRType::UInt64 || type.basetype == SPIRType::Int64) + require_extension_internal("GL_EXT_shader_atomic_int64"); + forced_temporaries.insert(result_id); emit_op(result_type, result_id, - join(op, "(", to_non_uniform_aware_expression(op0), ", ", + join(op, "(", to_atomic_ptr_expression(op0), ", ", to_unpacked_expression(op1), ")"), false); flush_all_atomic_capable_variables(); } @@ -7129,29 +7854,29 @@ string CompilerGLSL::legacy_tex_op(const std::string &op, const SPIRType &imgtyp const char *type; switch (imgtype.image.dim) { - case spv::Dim1D: + case Dim1D: // Force 2D path for ES. if (options.es) type = (imgtype.image.arrayed && !options.es) ? "2DArray" : "2D"; else type = (imgtype.image.arrayed && !options.es) ? "1DArray" : "1D"; break; - case spv::Dim2D: + case Dim2D: type = (imgtype.image.arrayed && !options.es) ? "2DArray" : "2D"; break; - case spv::Dim3D: + case Dim3D: type = "3D"; break; - case spv::DimCube: + case DimCube: type = "Cube"; break; - case spv::DimRect: + case DimRect: type = "2DRect"; break; - case spv::DimBuffer: + case DimBuffer: type = "Buffer"; break; - case spv::DimSubpassData: + case DimSubpassData: type = "2D"; break; default: @@ -7194,7 +7919,7 @@ string CompilerGLSL::legacy_tex_op(const std::string &op, const SPIRType &imgtyp else SPIRV_CROSS_THROW(join(op, " not allowed on depth samplers in legacy ES")); - if (imgtype.image.dim == spv::DimCube) + if (imgtype.image.dim == DimCube) return "shadowCubeNV"; } @@ -7461,7 +8186,7 @@ string CompilerGLSL::to_combined_image_sampler(VariableID image_id, VariableID s } } -bool CompilerGLSL::is_supported_subgroup_op_in_opengl(spv::Op op, const uint32_t *ops) +bool CompilerGLSL::is_supported_subgroup_op_in_opengl(Op op, const uint32_t *ops) { switch (op) { @@ -7641,7 +8366,7 @@ std::string CompilerGLSL::to_texture_op(const Instruction &i, bool sparse, bool auto &result_type = get(result_type_id); inherited_expressions.push_back(coord); - if (has_decoration(img, DecorationNonUniform) && !maybe_get_backing_variable(img)) + if (is_descriptor_non_uniform(img) && !maybe_get_backing_variable(img)) nonuniform_expression = true; switch (op) @@ -7695,6 +8420,9 @@ std::string CompilerGLSL::to_texture_op(const Instruction &i, bool sparse, bool case OpImageFetch: case OpImageSparseFetch: + if (options.vulkan_semantics && !dummy_sampler_id && (op == OpImageFetch || op == OpImageSparseFetch)) + require_extension_internal("GL_EXT_samplerless_texture_functions"); + // fallthrough case OpImageRead: // Reads == fetches in Metal (other langs will not get here) opt = &ops[4]; length -= 4; @@ -7723,19 +8451,19 @@ std::string CompilerGLSL::to_texture_op(const Instruction &i, bool sparse, bool uint32_t coord_components = 0; switch (imgtype.image.dim) { - case spv::Dim1D: + case Dim1D: coord_components = 1; break; - case spv::Dim2D: + case Dim2D: coord_components = 2; break; - case spv::Dim3D: + case Dim3D: coord_components = 3; break; - case spv::DimCube: + case DimCube: coord_components = 3; break; - case spv::DimBuffer: + case DimBuffer: coord_components = 1; break; default: @@ -7795,6 +8523,17 @@ std::string CompilerGLSL::to_texture_op(const Instruction &i, bool sparse, bool base_args.is_proj = proj != 0; string expr; + + // texture() with bias on sampler2DArrayShadow or samplerCubeArrayShadow requires GL_EXT_texture_shadow_lod. + // textureOffset() with bias on sampler2DArrayShadow also requires it. + if (bias != 0 && dref != 0 && !fetch && !gather && + ((imgtype.image.arrayed && imgtype.image.dim == Dim2D) || + (imgtype.image.arrayed && imgtype.image.dim == DimCube)) && + is_depth_image(imgtype, img)) + { + require_extension_internal("GL_EXT_texture_shadow_lod"); + } + TextureFunctionNameArguments name_args = {}; name_args.base = base_args; @@ -7820,6 +8559,7 @@ std::string CompilerGLSL::to_texture_op(const Instruction &i, bool sparse, bool args.grad_x = grad_x; args.grad_y = grad_y; args.lod = lod; + args.has_array_offsets = coffsets != 0; if (coffsets) args.offset = coffsets; @@ -7887,20 +8627,25 @@ bool CompilerGLSL::expression_is_constant_null(uint32_t id) const return c->constant_is_null(); } -bool CompilerGLSL::expression_is_non_value_type_array(uint32_t ptr) +bool CompilerGLSL::expression_is_non_value_type_array(uint32_t value_type_id, uint32_t ptr) { - auto &type = expression_type(ptr); - if (!is_array(get_pointee_type(type))) + auto &type = get(value_type_id); + if (!is_array(type)) return false; if (!backend.array_is_value_type) return true; + if (!backend.array_is_value_type_in_buffer_blocks && maybe_get_backing_buffer_pointer(ptr)) + return true; + auto *var = maybe_get_backing_variable(ptr); if (!var) return false; auto &backed_type = get(var->basetype); + + // Only consider explicitly laid out types here, not IO blocks. return !backend.array_is_value_type_in_buffer_blocks && backed_type.basetype == SPIRType::Struct && has_member_decoration(backed_type.self, 0, DecorationOffset); } @@ -7930,10 +8675,12 @@ string CompilerGLSL::to_function_name(const TextureFunctionNameArguments &args) { if (!expression_is_constant_null(args.lod)) { - SPIRV_CROSS_THROW("textureLod on sampler2DArrayShadow is not constant 0.0. This cannot be " - "expressed in GLSL."); + require_extension_internal("GL_EXT_texture_shadow_lod"); + } + else + { + workaround_lod_array_shadow_as_grad = true; } - workaround_lod_array_shadow_as_grad = true; } if (args.is_sparse_feedback) @@ -8068,9 +8815,11 @@ string CompilerGLSL::to_function_args(const TextureFunctionArguments &args, bool // To emulate this, we will have to use textureGrad with a constant gradient of 0. // The workaround will assert that the LOD is in fact constant 0, or we cannot emit correct code. // This happens for HLSL SampleCmpLevelZero on Texture2DArray and TextureCube. + // If GL_EXT_texture_shadow_lod is in use, textureLod is available directly with arbitrary LOD. bool workaround_lod_array_shadow_as_grad = ((imgtype.image.arrayed && imgtype.image.dim == Dim2D) || imgtype.image.dim == DimCube) && - is_depth_image(imgtype, img) && args.lod != 0 && !args.base.is_fetch; + is_depth_image(imgtype, img) && args.lod != 0 && !args.base.is_fetch && + !has_extension("GL_EXT_texture_shadow_lod"); if (args.dref) { @@ -8837,23 +9586,97 @@ void CompilerGLSL::emit_glsl_op(uint32_t result_type, uint32_t id, uint32_t eop, case GLSLstd450NMin: case GLSLstd450NMax: { - emit_nminmax_op(result_type, id, args[0], args[1], op); + if (options.vulkan_semantics) + { + require_extension_internal("GL_EXT_spirv_intrinsics"); + bool relaxed = has_decoration(id, DecorationRelaxedPrecision); + Polyfill poly = {}; + switch (get(result_type).width) + { + case 16: + poly = op == GLSLstd450NMin ? PolyfillNMin16 : PolyfillNMax16; + break; + + case 32: + poly = op == GLSLstd450NMin ? PolyfillNMin32 : PolyfillNMax32; + break; + + case 64: + poly = op == GLSLstd450NMin ? PolyfillNMin64 : PolyfillNMax64; + break; + + default: + SPIRV_CROSS_THROW("Invalid bit width for NMin/NMax."); + } + + require_polyfill(poly, relaxed); + + // Function return decorations are broken, so need to do double polyfill. + if (relaxed) + require_polyfill(poly, false); + + const char *op_str; + if (relaxed) + op_str = op == GLSLstd450NMin ? "spvNMinRelaxed" : "spvNMaxRelaxed"; + else + op_str = op == GLSLstd450NMin ? "spvNMin" : "spvNMax"; + + emit_binary_func_op(result_type, id, args[0], args[1], op_str); + } + else + { + emit_nminmax_op(result_type, id, args[0], args[1], op); + } break; } case GLSLstd450NClamp: { - // Make sure we have a unique ID here to avoid aliasing the extra sub-expressions between clamp and NMin sub-op. - // IDs cannot exceed 24 bits, so we can make use of the higher bits for some unique flags. - uint32_t &max_id = extra_sub_expressions[id | EXTRA_SUB_EXPRESSION_TYPE_AUX]; - if (!max_id) - max_id = ir.increase_bound_by(1); + if (options.vulkan_semantics) + { + require_extension_internal("GL_EXT_spirv_intrinsics"); + bool relaxed = has_decoration(id, DecorationRelaxedPrecision); + Polyfill poly = {}; + switch (get(result_type).width) + { + case 16: + poly = PolyfillNClamp16; + break; - // Inherit precision qualifiers. - ir.meta[max_id] = ir.meta[id]; + case 32: + poly = PolyfillNClamp32; + break; - emit_nminmax_op(result_type, max_id, args[0], args[1], GLSLstd450NMax); - emit_nminmax_op(result_type, id, max_id, args[2], GLSLstd450NMin); + case 64: + poly = PolyfillNClamp64; + break; + + default: + SPIRV_CROSS_THROW("Invalid bit width for NMin/NMax."); + } + + require_polyfill(poly, relaxed); + + // Function return decorations are broken, so need to do double polyfill. + if (relaxed) + require_polyfill(poly, false); + + emit_trinary_func_op(result_type, id, args[0], args[1], args[2], relaxed ? "spvNClampRelaxed" : "spvNClamp"); + } + else + { + // Make sure we have a unique ID here to avoid aliasing the extra sub-expressions between clamp and NMin sub-op. + // IDs cannot exceed 24 bits, so we can make use of the higher bits for some unique flags. + uint32_t &max_id = extra_sub_expressions[id | EXTRA_SUB_EXPRESSION_TYPE_AUX]; + if (!max_id) + max_id = ir.increase_bound_by(1); + + // Inherit precision qualifiers. + ir.meta[max_id] = ir.meta[id]; + + emit_nminmax_op(result_type, max_id, args[0], args[1], GLSLstd450NMax); + emit_nminmax_op(result_type, id, max_id, args[2], GLSLstd450NMin); + } break; } @@ -9101,6 +9924,35 @@ void CompilerGLSL::emit_subgroup_op(const Instruction &i) auto int_type = to_signed_basetype(integer_width); auto uint_type = to_unsigned_basetype(integer_width); + if (options.vulkan_semantics) + { + auto &return_type = get(ops[0]); + switch (return_type.basetype) + { + case SPIRType::SByte: + case SPIRType::UByte: + require_extension_internal("GL_EXT_shader_subgroup_extended_types_int8"); + break; + + case SPIRType::Short: + case SPIRType::UShort: + require_extension_internal("GL_EXT_shader_subgroup_extended_types_int16"); + break; + + case SPIRType::Half: + require_extension_internal("GL_EXT_shader_subgroup_extended_types_float16"); + break; + + case SPIRType::Int64: + case SPIRType::UInt64: + require_extension_internal("GL_EXT_shader_subgroup_extended_types_int64"); + break; + + default: + break; + } + } + switch (op) { case OpGroupNonUniformElect: @@ -9149,6 +10001,10 @@ void CompilerGLSL::emit_subgroup_op(const Instruction &i) require_extension_internal("GL_KHR_shader_subgroup_shuffle_relative"); break; + case OpGroupNonUniformRotateKHR: + require_extension_internal("GL_KHR_shader_subgroup_rotate"); + break; + case OpGroupNonUniformAll: case OpGroupNonUniformAny: case OpGroupNonUniformAllEqual: @@ -9220,6 +10076,13 @@ void CompilerGLSL::emit_subgroup_op(const Instruction &i) require_extension_internal("GL_KHR_shader_subgroup_quad"); break; + case OpGroupNonUniformQuadAllKHR: + case OpGroupNonUniformQuadAnyKHR: + // Require both extensions to be enabled. + require_extension_internal("GL_KHR_shader_subgroup_vote"); + require_extension_internal("GL_EXT_shader_quad_control"); + break; + default: SPIRV_CROSS_THROW("Invalid opcode for subgroup."); } @@ -9227,9 +10090,13 @@ void CompilerGLSL::emit_subgroup_op(const Instruction &i) uint32_t result_type = ops[0]; uint32_t id = ops[1]; - auto scope = static_cast(evaluate_constant_u32(ops[2])); - if (scope != ScopeSubgroup) - SPIRV_CROSS_THROW("Only subgroup scope is supported."); + // These quad ops do not have a scope parameter. + if (op != OpGroupNonUniformQuadAllKHR && op != OpGroupNonUniformQuadAnyKHR) + { + auto scope = static_cast(evaluate_constant_u32(ops[2])); + if (scope != ScopeSubgroup) + SPIRV_CROSS_THROW("Only subgroup scope is supported."); + } switch (op) { @@ -9295,6 +10162,13 @@ void CompilerGLSL::emit_subgroup_op(const Instruction &i) emit_binary_func_op(result_type, id, ops[3], ops[4], "subgroupShuffleDown"); break; + case OpGroupNonUniformRotateKHR: + if (i.length > 5) + emit_trinary_func_op(result_type, id, ops[3], ops[4], ops[5], "subgroupClusteredRotate"); + else + emit_binary_func_op(result_type, id, ops[3], ops[4], "subgroupRotate"); + break; + case OpGroupNonUniformAll: emit_unary_func_op(result_type, id, ops[3], "subgroupAll"); break; @@ -9382,6 +10256,14 @@ case OpGroupNonUniform##op: \ break; } + case OpGroupNonUniformQuadAllKHR: + emit_unary_func_op(result_type, id, ops[2], "subgroupQuadAll"); + break; + + case OpGroupNonUniformQuadAnyKHR: + emit_unary_func_op(result_type, id, ops[2], "subgroupQuadAny"); + break; + default: SPIRV_CROSS_THROW("Invalid opcode for subgroup."); } @@ -9497,6 +10379,30 @@ string CompilerGLSL::bitcast_glsl_op(const SPIRType &out_type, const SPIRType &i return "packUint4x16"; else if (out_type.basetype == SPIRType::UShort && in_type.basetype == SPIRType::UInt64 && in_type.vecsize == 1) return "unpackUint4x16"; + else if (out_type.basetype == SPIRType::BFloat16 && in_type.basetype == SPIRType::UShort) + return "uintBitsToBFloat16EXT"; + else if (out_type.basetype == SPIRType::BFloat16 && in_type.basetype == SPIRType::Short) + return "intBitsToBFloat16EXT"; + else if (out_type.basetype == SPIRType::UShort && in_type.basetype == SPIRType::BFloat16) + return "bfloat16BitsToUintEXT"; + else if (out_type.basetype == SPIRType::Short && in_type.basetype == SPIRType::BFloat16) + return "bfloat16BitsToIntEXT"; + else if (out_type.basetype == SPIRType::FloatE4M3 && in_type.basetype == SPIRType::UByte) + return "uintBitsToFloate4m3EXT"; + else if (out_type.basetype == SPIRType::FloatE4M3 && in_type.basetype == SPIRType::SByte) + return "intBitsToFloate4m3EXT"; + else if (out_type.basetype == SPIRType::UByte && in_type.basetype == SPIRType::FloatE4M3) + return "floate4m3BitsToUintEXT"; + else if (out_type.basetype == SPIRType::SByte && in_type.basetype == SPIRType::FloatE4M3) + return "floate4m3BitsToIntEXT"; + else if (out_type.basetype == SPIRType::FloatE5M2 && in_type.basetype == SPIRType::UByte) + return "uintBitsToFloate5m2EXT"; + else if (out_type.basetype == SPIRType::FloatE5M2 && in_type.basetype == SPIRType::SByte) + return "intBitsToFloate5m2EXT"; + else if (out_type.basetype == SPIRType::UByte && in_type.basetype == SPIRType::FloatE5M2) + return "floate5m2BitsToUintEXT"; + else if (out_type.basetype == SPIRType::SByte && in_type.basetype == SPIRType::FloatE5M2) + return "floate5m2BitsToIntEXT"; return ""; } @@ -9566,9 +10472,9 @@ string CompilerGLSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) auto model = get_entry_point().model; switch (model) { - case spv::ExecutionModelIntersectionKHR: - case spv::ExecutionModelAnyHitKHR: - case spv::ExecutionModelClosestHitKHR: + case ExecutionModelIntersectionKHR: + case ExecutionModelAnyHitKHR: + case ExecutionModelClosestHitKHR: // gl_InstanceID is allowed in these shaders. break; @@ -9615,7 +10521,17 @@ string CompilerGLSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) case BuiltInInvocationId: return "gl_InvocationID"; case BuiltInLayer: + { + auto model = get_execution_model(); + if (model == ExecutionModelVertex || model == ExecutionModelTessellationEvaluation) + { + if (options.es) + require_extension_internal("GL_NV_viewport_array2"); + else + require_extension_internal("GL_ARB_shader_viewport_layer_array"); + } return "gl_Layer"; + } case BuiltInViewportIndex: return "gl_ViewportIndex"; case BuiltInTessLevelOuter: @@ -9890,6 +10806,31 @@ string CompilerGLSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) case BuiltInCullPrimitiveEXT: return "gl_CullPrimitiveEXT"; + case BuiltInHitTriangleVertexPositionsKHR: + { + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("Need Vulkan semantics for EXT_ray_tracing_position_fetch."); + require_extension_internal("GL_EXT_ray_tracing_position_fetch"); + return "gl_HitTriangleVertexPositionsEXT"; + } + + case BuiltInClusterIDNV: + { + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("Can only use ClusterIDNV in Vulkan GLSL."); + require_extension_internal("GL_NV_cluster_acceleration_structure"); + return "gl_ClusterIDNV"; + } + + case BuiltInResourceHeapEXT: + // This builtin name is a placeholder. + // We will override this name later with prefix per actual type. + // However, this allows untyped access chain to index into the heap directly. + return "ResourceHeap"; + + case BuiltInSamplerHeapEXT: + return "SamplerHeap"; + default: return join("gl_BuiltIn_", convert_to_string(builtin)); } @@ -9951,7 +10892,8 @@ bool CompilerGLSL::access_chain_needs_stage_io_builtin_translation(uint32_t) } string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indices, uint32_t count, - AccessChainFlags flags, AccessChainMeta *meta) + AccessChainFlags flags, AccessChainMeta *meta, + const SPIRType *untyped_data_type) { string expr; @@ -9977,7 +10919,12 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice // Start traversing type hierarchy at the proper non-pointer types, // but keep type_id referencing the original pointer for use below. uint32_t type_id = expression_type_id(base); - const auto *type = &get_pointee_type(type_id); + + // If nullptr we're doing untyped pointers. + // For now we don't really care about types since we're just doing a single index into the heap. + // If we intend to support complete untyped pointers usage later, we need to pass down the base type + // and override chain type based on that. + const auto *type = untyped_data_type ? untyped_data_type : &get_pointee_type(type_id); if (!backend.native_pointers) { @@ -10001,6 +10948,8 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice bool pending_array_enclose = false; bool dimension_flatten = false; bool access_meshlet_position_y = false; + bool chain_is_builtin = false; + BuiltIn chained_builtin = {}; if (auto *base_expr = maybe_get(base)) { @@ -10017,7 +10966,8 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice if (!is_ptr_chain) mod_flags &= ~ACCESS_CHAIN_PTR_CHAIN_BIT; access_chain_internal_append_index(expr, base, type, mod_flags, access_chain_is_arrayed, index); - check_physical_type_cast(expr, type, physical_type); + if (check_physical_type_cast(expr, type, physical_type)) + physical_type = 0; }; for (uint32_t i = 0; i < count; i++) @@ -10080,7 +11030,40 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice } else { - append_index(index, is_literal, true); + if (flags & ACCESS_CHAIN_PTR_CHAIN_POINTER_ARITH_BIT) + { + SPIRType tmp_type(OpTypeInt); + tmp_type.basetype = SPIRType::UInt64; + tmp_type.width = 64; + tmp_type.vecsize = 1; + tmp_type.columns = 1; + + TypeID ptr_type_id = expression_type_id(base); + const SPIRType &ptr_type = get(ptr_type_id); + const SPIRType &pointee_type = get_pointee_type(ptr_type); + + // This only runs in native pointer backends. + // Can replace reinterpret_cast with a backend string if ever needed. + // We expect this to count as a de-reference. + // This leaks some MSL details, but feels slightly overkill to + // add yet another virtual interface just for this. + auto intptr_expr = join("reinterpret_cast<", type_to_glsl(tmp_type), ">(", expr, ")"); + intptr_expr += join(" + ", to_enclosed_unpacked_expression(index), " * ", + get_decoration(ptr_type_id, DecorationArrayStride)); + + if (flags & ACCESS_CHAIN_PTR_CHAIN_CAST_TO_SCALAR_BIT) + { + is_packed = true; + expr = join("*reinterpret_cast(", intptr_expr, ")"); + } + else + { + expr = join("*reinterpret_cast<", type_to_glsl(ptr_type), ">(", intptr_expr, ")"); + } + } + else + append_index(index, is_literal, true); } if (type->basetype == SPIRType::ControlPointArray) @@ -10097,8 +11080,8 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice if (ptr_chain_array_entry) expr = join("(", expr, ")"); } - // Arrays - else if (!type->array.empty()) + // Arrays and OpTypeCooperativeVectorNV (aka fancy arrays) + else if (!type->array.empty() || type->op == OpTypeCooperativeVectorNV) { // If we are flattening multidimensional arrays, only create opening bracket on first // array index. @@ -10125,6 +11108,9 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice auto builtin = ir.meta[base].decoration.builtin_type; bool mesh_shader = get_execution_model() == ExecutionModelMeshEXT; + chain_is_builtin = true; + chained_builtin = builtin; + switch (builtin) { case BuiltInCullDistance: @@ -10210,6 +11196,15 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice access_meshlet_position_y = true; } + if (get(type->parent_type).op == OpTypeStruct && + has_decoration(type->parent_type, DecorationArrayStride)) + { + uint32_t native_stride = get_decoration(type->parent_type, DecorationArrayStride); + uint32_t array_stride = get_decoration(type_id, DecorationArrayStride); + if (native_stride != array_stride) + expr += ".data"; + } + type_id = type->parent_type; type = &get(type_id); @@ -10260,6 +11255,9 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice { access_meshlet_position_y = true; } + + chain_is_builtin = true; + chained_builtin = builtin; } else { @@ -10292,6 +11290,7 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice physical_type = 0; row_major_matrix_needs_conversion = member_is_non_native_row_major_matrix(*type, index); + type_id = type->member_types[index]; type = &get(type->member_types[index]); } // Matrix -> Vector @@ -10322,7 +11321,7 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice type = &get(type_id); } // Vector -> Scalar - else if (type->vecsize > 1) + else if (type->op == OpTypeCooperativeMatrixKHR || type->vecsize > 1) { string deferred_index; if (row_major_matrix_needs_conversion) @@ -10384,9 +11383,9 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice if (is_literal) { - bool out_of_bounds = (index >= type->vecsize); + bool out_of_bounds = index >= type->vecsize && type->op != OpTypeCooperativeMatrixKHR; - if (!is_packed && !row_major_matrix_needs_conversion) + if (!is_packed && !row_major_matrix_needs_conversion && type->op != OpTypeCooperativeMatrixKHR) { expr += "."; expr += index_to_swizzle(out_of_bounds ? 0 : index); @@ -10479,16 +11478,19 @@ string CompilerGLSL::access_chain_internal(uint32_t base, const uint32_t *indice meta->storage_physical_type = physical_type; meta->relaxed_precision = relaxed_precision; meta->access_meshlet_position_y = access_meshlet_position_y; + meta->chain_is_builtin = chain_is_builtin; + meta->builtin = chained_builtin; } return expr; } -void CompilerGLSL::check_physical_type_cast(std::string &, const SPIRType *, uint32_t) +bool CompilerGLSL::check_physical_type_cast(std::string &, const SPIRType *, uint32_t) { + return false; } -bool CompilerGLSL::prepare_access_chain_for_scalar_access(std::string &, const SPIRType &, spv::StorageClass, bool &) +bool CompilerGLSL::prepare_access_chain_for_scalar_access(std::string &, const SPIRType &, StorageClass, bool &) { return false; } @@ -10500,11 +11502,19 @@ string CompilerGLSL::to_flattened_struct_member(const string &basename, const SP return ret; } +uint32_t CompilerGLSL::get_physical_type_id_stride(TypeID) const +{ + SPIRV_CROSS_THROW("Invalid to call get_physical_type_id_stride on a backend without native pointer support."); +} + string CompilerGLSL::access_chain(uint32_t base, const uint32_t *indices, uint32_t count, const SPIRType &target_type, - AccessChainMeta *meta, bool ptr_chain) + AccessChainMeta *meta, bool ptr_chain, const SPIRType *untyped_data_type) { if (flattened_buffer_blocks.count(base)) { + if (untyped_data_type) + SPIRV_CROSS_THROW("Flattening not compatible with untyped pointers."); + uint32_t matrix_stride = 0; uint32_t array_stride = 0; bool need_transpose = false; @@ -10522,6 +11532,9 @@ string CompilerGLSL::access_chain(uint32_t base, const uint32_t *indices, uint32 } else if (flattened_structs.count(base) && count > 0) { + if (untyped_data_type) + SPIRV_CROSS_THROW("Flattening not compatible with untyped pointers."); + AccessChainFlags flags = ACCESS_CHAIN_CHAIN_ONLY_BIT | ACCESS_CHAIN_SKIP_REGISTER_EXPRESSION_READ_BIT; if (ptr_chain) flags |= ACCESS_CHAIN_PTR_CHAIN_BIT; @@ -10533,7 +11546,7 @@ string CompilerGLSL::access_chain(uint32_t base, const uint32_t *indices, uint32 meta->flattened_struct = target_type.basetype == SPIRType::Struct; } - auto chain = access_chain_internal(base, indices, count, flags, nullptr).substr(1); + auto chain = access_chain_internal(base, indices, count, flags, nullptr, nullptr).substr(1); if (meta) { meta->need_transpose = false; @@ -10549,8 +11562,28 @@ string CompilerGLSL::access_chain(uint32_t base, const uint32_t *indices, uint32 { AccessChainFlags flags = ACCESS_CHAIN_SKIP_REGISTER_EXPRESSION_READ_BIT; if (ptr_chain) + { flags |= ACCESS_CHAIN_PTR_CHAIN_BIT; - return access_chain_internal(base, indices, count, flags, meta); + // PtrAccessChain could get complicated. + TypeID type_id = expression_type_id(base); + if (backend.native_pointers && has_decoration(type_id, DecorationArrayStride)) + { + // If there is a mismatch we have to go via 64-bit pointer arithmetic :'( + // Using packed hacks only gets us so far, and is not designed to deal with pointer to + // random values. It works for structs though. + TypeID pointee_type_id = get_pointee_type_id(type_id); + uint32_t physical_stride = get_physical_type_id_stride(pointee_type_id); + uint32_t requested_stride = get_decoration(type_id, DecorationArrayStride); + if (physical_stride != requested_stride) + { + flags |= ACCESS_CHAIN_PTR_CHAIN_POINTER_ARITH_BIT; + if (is_vector(get(pointee_type_id))) + flags |= ACCESS_CHAIN_PTR_CHAIN_CAST_TO_SCALAR_BIT; + } + } + } + + return access_chain_internal(base, indices, count, flags, meta, untyped_data_type); } } @@ -10962,7 +11995,7 @@ bool CompilerGLSL::should_dereference(uint32_t id) { const auto &type = expression_type(id); // Non-pointer expressions don't need to be dereferenced. - if (!type.pointer) + if (!is_pointer(type)) return false; // Handles shouldn't be dereferenced either. @@ -10970,8 +12003,9 @@ bool CompilerGLSL::should_dereference(uint32_t id) return false; // If id is a variable but not a phi variable, we should not dereference it. + // BDA passed around as parameters are always pointers. if (auto *var = maybe_get(id)) - return var->phi_variable; + return (var->parameter && is_physical_or_buffer_pointer(type)) || var->phi_variable; if (auto *expr = maybe_get(id)) { @@ -11004,6 +12038,16 @@ bool CompilerGLSL::should_dereference(uint32_t id) return true; } +bool CompilerGLSL::should_dereference_caller_param(uint32_t id) +{ + const auto &type = expression_type(id); + // BDA is always passed around as pointers. Similarly, we need to pass variable buffer pointers as pointers. + if (is_physical_or_buffer_pointer(type)) + return false; + + return should_dereference(id); +} + bool CompilerGLSL::should_forward(uint32_t id) const { // If id is a variable we will try to forward it regardless of force_temporary check below @@ -11039,6 +12083,9 @@ bool CompilerGLSL::should_forward(uint32_t id) const if (is_immutable(id)) return true; + if (expr && expr->buffer_pointer) + return true; + return false; } @@ -11127,6 +12174,8 @@ void CompilerGLSL::register_impure_function_call() flush_dependees(get(global)); for (auto aliased : aliased_variables) flush_dependees(get(aliased)); + for (auto ptr : buffer_pointer_variables) + flush_dependees(get(ptr)); } void CompilerGLSL::register_call_out_argument(uint32_t id) @@ -11158,7 +12207,7 @@ void CompilerGLSL::emit_variable_temporary_copies(const SPIRVariable &var) { auto &type = get(var.basetype); auto &flags = get_decoration_bitset(var.self); - statement(flags_to_qualifiers_glsl(type, flags), variable_decl(type, join("_", var.self, "_copy")), ";"); + statement(flags_to_qualifiers_glsl(type, var.self, flags), variable_decl(type, join("_", var.self, "_copy")), ";"); flushed_phi_variables.insert(var.self); } } @@ -11298,7 +12347,8 @@ string CompilerGLSL::build_composite_combiner(uint32_t return_type, const uint32 // Can only merge swizzles for vectors. auto &type = get(return_type); - bool can_apply_swizzle_opt = type.basetype != SPIRType::Struct && type.array.empty() && type.columns == 1; + bool can_apply_swizzle_opt = type.basetype != SPIRType::Struct && type.array.empty() && type.columns == 1 && + type.op != OpTypeCooperativeMatrixKHR; bool swizzle_optimization = false; for (uint32_t i = 0; i < length; i++) @@ -11499,13 +12549,13 @@ void CompilerGLSL::disallow_forwarding_in_expression_chain(const SPIRExpression // Allow trivially forwarded expressions like OpLoad or trivial shuffles, // these will be marked as having suppressed usage tracking. // Our only concern is to make sure arithmetic operations are done in similar ways. - if (expression_is_forwarded(expr.self) && !expression_suppresses_usage_tracking(expr.self) && - forced_invariant_temporaries.count(expr.self) == 0) + if (forced_invariant_temporaries.count(expr.self) == 0) { - force_temporary_and_recompile(expr.self); + if (!expression_suppresses_usage_tracking(expr.self)) + force_temporary_and_recompile(expr.self); forced_invariant_temporaries.insert(expr.self); - for (auto &dependent : expr.expression_dependencies) + for (auto &dependent : expr.invariance_dependencies) disallow_forwarding_in_expression_chain(get(dependent)); } } @@ -11541,7 +12591,7 @@ void CompilerGLSL::emit_store_statement(uint32_t lhs_expression, uint32_t rhs_ex if (!unroll_array_to_complex_store(lhs_expression, rhs_expression)) { auto lhs = to_dereferenced_expression(lhs_expression); - if (has_decoration(lhs_expression, DecorationNonUniform)) + if (is_descriptor_non_uniform(lhs_expression)) convert_non_uniform_expression(lhs, lhs_expression); // We might need to cast in order to store to a builtin. @@ -11583,6 +12633,10 @@ uint32_t CompilerGLSL::get_integer_width_for_instruction(const Instruction &inst case OpUGreaterThanEqual: return expression_type(ops[2]).width; + case OpSMulExtended: + case OpUMulExtended: + return get(get(ops[0]).member_types[0]).width; + default: { // We can look at result type which is more robust. @@ -11755,6 +12809,18 @@ static bool opcode_is_precision_sensitive_operation(Op op) case OpConvertUToF: case OpConvertFToU: case OpConvertFToS: + case OpShiftLeftLogical: + case OpShiftRightLogical: + case OpShiftRightArithmetic: + case OpBitwiseOr: + case OpBitwiseXor: + case OpBitwiseAnd: + case OpNot: + case OpBitFieldInsert: + case OpBitFieldSExtract: + case OpBitFieldUExtract: + case OpBitReverse: + case OpBitCount: return true; default: @@ -11851,6 +12917,33 @@ CompilerGLSL::TemporaryCopy CompilerGLSL::handle_instruction_precision(const Ins return {}; } +static pair split_coopmat_pointer(const string &expr) +{ + auto ptr_expr = expr; + string index_expr; + + if (ptr_expr.back() != ']') + SPIRV_CROSS_THROW("Access chain for coopmat must be indexed into an array."); + + // Strip the access chain. + ptr_expr.pop_back(); + uint32_t counter = 1; + while (counter && !ptr_expr.empty()) + { + if (ptr_expr.back() == ']') + counter++; + else if (ptr_expr.back() == '[') + counter--; + ptr_expr.pop_back(); + } + + if (ptr_expr.empty()) + SPIRV_CROSS_THROW("Invalid pointer expression for coopmat."); + + index_expr = expr.substr(ptr_expr.size() + 1, expr.size() - (ptr_expr.size() + 1) - 1); + return { std::move(ptr_expr), std::move(index_expr) }; +} + void CompilerGLSL::emit_instruction(const Instruction &instruction) { auto ops = stream(instruction); @@ -11901,6 +12994,14 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // If an expression is mutable and forwardable, we speculate that it is immutable. bool forward = should_forward(ptr) && forced_temporaries.find(id) == end(forced_temporaries); + // Volatile memory access requires the value be read exactly once from + // memory. Do not forward the expression so that re-evaluation at each + // use site cannot re-read potentially modified memory. + // FIXME: To force implementations to actually respect the volatile nature of the load, + // the block itself must be marked volatile, or VulkanMM is used to do an explicit volatile load. + if (forward && length >= 4 && (ops[3] & MemoryAccessVolatileMask) != 0) + forward = false; + // If loading a non-native row-major matrix, mark the expression as need_transpose. bool need_transpose = false; bool old_need_transpose = false; @@ -11964,7 +13065,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // Also, loading from gl_SampleMask array needs special unroll. unroll_array_from_complex_load(id, ptr, expr); - if (!type_is_opaque_value(type) && has_decoration(ptr, DecorationNonUniform)) + if (!type_is_opaque_value(type) && is_descriptor_non_uniform(ptr)) { // If we're loading something non-opaque, we need to handle non-uniform descriptor access. convert_non_uniform_expression(expr, ptr); @@ -11984,7 +13085,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) bool usage_tracking = flattened && (type.basetype == SPIRType::Struct || (type.columns > 1)); SPIRExpression *e = nullptr; - if (!forward && expression_is_non_value_type_array(ptr)) + if (!forward && expression_is_non_value_type_array(result_type, ptr)) { // Complicated load case where we need to make a copy of ptr, but we cannot, because // it is an array, and our backend does not support arrays as value types. @@ -12022,11 +13123,41 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpUntypedPtrAccessChainKHR: + SPIRV_CROSS_THROW("OpUntypedPtrAccessChainKHR is not supported."); + break; + + case OpUntypedAccessChainKHR: + case OpUntypedInBoundsAccessChainKHR: case OpInBoundsAccessChain: case OpAccessChain: case OpPtrAccessChain: { - auto *var = maybe_get(ops[2]); + bool untyped = opcode == OpUntypedAccessChainKHR || opcode == OpUntypedInBoundsAccessChainKHR; + + uint32_t type_id = ops[0]; + uint32_t result_id = ops[1]; + uint32_t ptr_id = ops[untyped ? 3 : 2]; + uint32_t indices_start = untyped ? 4 : 3; + + if (untyped) + { + auto *var = maybe_get_backing_variable(ptr_id); + // Buffer pointers stop the loaded from chain to deal with aliasing better, so carve that out specifically. + auto *expr = maybe_get_backing_buffer_pointer(ptr_id); + + if (!expr) + { + if (!var || !has_decoration(var->self, DecorationBuiltIn) || + (BuiltIn(get_decoration(var->self, DecorationBuiltIn)) != BuiltInResourceHeapEXT && + BuiltIn(get_decoration(var->self, DecorationBuiltIn)) != BuiltInSamplerHeapEXT)) + { + SPIRV_CROSS_THROW("Untyped pointer access chains are currently only supported for descriptor heap access."); + } + } + } + + auto *var = maybe_get(ptr_id); if (var) flush_variable_declaration(var->self); @@ -12034,55 +13165,78 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // If an expression is mutable and forwardable, we speculate that it is immutable. AccessChainMeta meta; bool ptr_chain = opcode == OpPtrAccessChain; - auto &target_type = get(ops[0]); - auto e = access_chain(ops[2], &ops[3], length - 3, target_type, &meta, ptr_chain); + auto &target_type = get(type_id); + auto e = access_chain(ptr_id, &ops[indices_start], length - indices_start, target_type, &meta, ptr_chain, + untyped ? &get(ops[2]) : nullptr); + + if (untyped) + { + auto &data_type = get(ops[2]); + auto *ptr_expr = maybe_get(ptr_id); + if (data_type.basetype == SPIRType::Image || data_type.basetype == SPIRType::Sampler || + data_type.basetype == SPIRType::AccelerationStructure || + (ptr_expr && ptr_expr->buffer_pointer)) + { + // We can resolve this type now. + // For further buffer access chains, we don't do any fixups since we have resolved to proper types. + // For buffer types we only prepend when the access chain starts from a BufferPointerEXT base. + // Multi-stage access chains are not possible for image types. + if (ptr_expr && ptr_expr->buffer_pointer) + e = join(to_buffer_pointer_name_prefix(ptr_expr->self), e); + else + e = join("spv", to_name(data_type.self), e); + } + } // If the base is flattened UBO of struct type, the expression has to be a composite. // In that case, backends which do not support inline syntax need it to be bound to a temporary. // Otherwise, invalid expressions like ({UBO[0].xyz, UBO[0].w, UBO[1]}).member are emitted. bool requires_temporary = false; - if (flattened_buffer_blocks.count(ops[2]) && target_type.basetype == SPIRType::Struct) + if (flattened_buffer_blocks.count(ptr_id) && target_type.basetype == SPIRType::Struct) requires_temporary = !backend.can_declare_struct_inline; auto &expr = requires_temporary ? - emit_op(ops[0], ops[1], std::move(e), false) : - set(ops[1], std::move(e), ops[0], should_forward(ops[2])); + emit_op(type_id, result_id, std::move(e), false) : + set(result_id, std::move(e), type_id, should_forward(ptr_id)); - auto *backing_variable = maybe_get_backing_variable(ops[2]); - expr.loaded_from = backing_variable ? backing_variable->self : ID(ops[2]); + auto *backing_variable = maybe_get_backing_variable(ptr_id); + expr.loaded_from = backing_variable ? backing_variable->self : ID(ptr_id); expr.need_transpose = meta.need_transpose; expr.access_chain = true; expr.access_meshlet_position_y = meta.access_meshlet_position_y; // Mark the result as being packed. Some platforms handled packed vectors differently than non-packed. if (meta.storage_is_packed) - set_extended_decoration(ops[1], SPIRVCrossDecorationPhysicalTypePacked); + set_extended_decoration(result_id, SPIRVCrossDecorationPhysicalTypePacked); if (meta.storage_physical_type != 0) - set_extended_decoration(ops[1], SPIRVCrossDecorationPhysicalTypeID, meta.storage_physical_type); + set_extended_decoration(result_id, SPIRVCrossDecorationPhysicalTypeID, meta.storage_physical_type); if (meta.storage_is_invariant) - set_decoration(ops[1], DecorationInvariant); + set_decoration(result_id, DecorationInvariant); if (meta.flattened_struct) - flattened_structs[ops[1]] = true; + flattened_structs[result_id] = true; if (meta.relaxed_precision && backend.requires_relaxed_precision_analysis) - set_decoration(ops[1], DecorationRelaxedPrecision); + set_decoration(result_id, DecorationRelaxedPrecision); + if (meta.chain_is_builtin) + set_decoration(result_id, DecorationBuiltIn, meta.builtin); // If we have some expression dependencies in our access chain, this access chain is technically a forwarded // temporary which could be subject to invalidation. // Need to assume we're forwarded while calling inherit_expression_depdendencies. - forwarded_temporaries.insert(ops[1]); + forwarded_temporaries.insert(result_id); // The access chain itself is never forced to a temporary, but its dependencies might. - suppressed_usage_tracking.insert(ops[1]); + suppressed_usage_tracking.insert(result_id); - for (uint32_t i = 2; i < length; i++) + // Include the base pointer. + for (uint32_t i = indices_start - 1; i < length; i++) { - inherit_expression_dependencies(ops[1], ops[i]); + inherit_expression_dependencies(result_id, ops[i]); add_implied_read_expression(expr, ops[i]); } // If we have no dependencies after all, i.e., all indices in the access chain are immutable temporaries, // we're not forwarded after all. if (expr.expression_dependencies.empty()) - forwarded_temporaries.erase(ops[1]); + forwarded_temporaries.erase(result_id); break; } @@ -12116,15 +13270,80 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpUntypedArrayLengthKHR: case OpArrayLength: { + bool untyped = opcode == OpUntypedArrayLengthKHR; uint32_t result_type = ops[0]; uint32_t id = ops[1]; - auto e = access_chain_internal(ops[2], &ops[3], length - 3, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr); - if (has_decoration(ops[2], DecorationNonUniform)) - convert_non_uniform_expression(e, ops[2]); - set(id, join(type_to_glsl(get(result_type)), "(", e, ".length())"), result_type, - true); + + const SPIRType *untyped_data_type = untyped ? &get(ops[2]) : nullptr; + uint32_t ptr_id = ops[untyped ? 3 : 2]; + uint32_t index_offset = untyped ? 4 : 3; + + auto e = access_chain_internal(ptr_id, &ops[index_offset], length - index_offset, + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, + nullptr, untyped_data_type); + + if (untyped) + { + auto *ptr_expr = maybe_get(ptr_id); + if (ptr_expr && ptr_expr->buffer_pointer) + e = join(to_buffer_pointer_name_prefix(ptr_expr->self), e); + } + + if (is_descriptor_non_uniform(ptr_id)) + convert_non_uniform_expression(e, ptr_id); + set(id, join(type_to_glsl(get(result_type)), "(", e, ".length())"), result_type, true); + break; + } + + case OpBufferPointerEXT: + { + uint32_t type_id = ops[0]; + uint32_t result_id = ops[1]; + uint32_t ptr_id = ops[2]; + + auto *backing_variable = maybe_get_backing_variable(ptr_id); + if (!backing_variable) + SPIRV_CROSS_THROW("There is no backing variable for BufferPointerEXT."); + + auto *chain_expr = maybe_get(ptr_id); + if (!chain_expr || !chain_expr->access_chain) + SPIRV_CROSS_THROW("Expected to see access chain for BufferPointerEXT."); + + auto e = to_expression(ptr_id); + + // BufferPointerEXT can return a typed pointer, in which case we need to resolve the heap alias now. + auto &type = get(type_id); + if (type.basetype == SPIRType::Struct) + e = join(to_buffer_pointer_name_prefix(result_id), e); + + auto &expr = set(result_id, std::move(e), type_id, true); + // There isn't any backing variable here. OpBufferPointerEXT is meant to be a memory declaration instruction. + expr.loaded_from = 0; + expr.access_chain = true; + expr.buffer_pointer = true; + expr.implied_read_expressions = chain_expr->implied_read_expressions; + expr.expression_dependencies = chain_expr->expression_dependencies; + expr.immutable = false; + + // If the buffer pointer is marked non-writable, ignore alias tracking by flagging the expression as immutable. + for (auto &heap : descriptor_heap_types) + { + if (heap.buffer_pointer_id == result_id) + { + if (heap.nonwritable) + expr.immutable = true; + break; + } + } + + if (!expr.immutable && ir.get_buffer_block_type_flags(get(type_id)).get(DecorationNonWritable)) + expr.immutable = true; + + // Used for load-store tracking. + buffer_pointer_variables.push_back(result_id); break; } @@ -12140,6 +13359,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) auto &callee = get(func); auto &return_type = get(callee.return_type); bool pure = function_is_pure(callee); + bool control_dependent = function_is_control_dependent(callee); bool callee_has_out_variables = false; bool emit_return_value_as_argument = false; @@ -12171,7 +13391,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) if (emit_return_value_as_argument) { - statement(type_to_glsl(return_type), " ", to_name(id), type_to_array_glsl(return_type), ";"); + statement(type_to_glsl(return_type), " ", to_name(id), type_to_array_glsl(return_type, 0), ";"); arglist.push_back(to_name(id)); } @@ -12233,6 +13453,9 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) else statement(funexpr, ";"); + if (control_dependent) + register_control_dependent_expression(id); + break; } @@ -12354,7 +13577,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // Make a copy, then use access chain to store the variable. statement(declare_temporary(result_type, id), to_expression(vec), ";"); set(id, to_name(id), result_type, true); - auto chain = access_chain_internal(id, &index, 1, 0, nullptr); + auto chain = access_chain_internal(id, &index, 1, 0, nullptr, nullptr); statement(chain, " = ", to_unpacked_expression(comp), ";"); break; } @@ -12364,7 +13587,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) uint32_t result_type = ops[0]; uint32_t id = ops[1]; - auto expr = access_chain_internal(ops[2], &ops[3], 1, 0, nullptr); + auto expr = access_chain_internal(ops[2], &ops[3], 1, 0, nullptr, nullptr); emit_op(result_type, id, expr, should_forward(ops[2])); inherit_expression_dependencies(id, ops[2]); inherit_expression_dependencies(id, ops[3]); @@ -12388,6 +13611,9 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) if (composite_type_is_complex) allow_base_expression = false; + if (composite_type.op == OpTypeCooperativeMatrixKHR) + allow_base_expression = false; + // Packed expressions or physical ID mapped expressions cannot be split up. if (has_extended_decoration(ops[2], SPIRVCrossDecorationPhysicalTypePacked) || has_extended_decoration(ops[2], SPIRVCrossDecorationPhysicalTypeID)) @@ -12425,7 +13651,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // from expression causing it to be forced to an actual temporary in GLSL. auto expr = access_chain_internal(ops[2], &ops[3], length, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_CHAIN_ONLY_BIT | - ACCESS_CHAIN_FORCE_COMPOSITE_BIT, &meta); + ACCESS_CHAIN_FORCE_COMPOSITE_BIT, &meta, nullptr); e = &emit_op(result_type, id, expr, true, should_suppress_usage_tracking(ops[2])); inherit_expression_dependencies(id, ops[2]); e->base_expression = ops[2]; @@ -12436,7 +13662,8 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) else { auto expr = access_chain_internal(ops[2], &ops[3], length, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_FORCE_COMPOSITE_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_FORCE_COMPOSITE_BIT, + &meta, nullptr); e = &emit_op(result_type, id, expr, should_forward(ops[2]), should_suppress_usage_tracking(ops[2])); inherit_expression_dependencies(id, ops[2]); } @@ -12504,7 +13731,8 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) if (!forced_temporaries.count(composite)) force_temporary_and_recompile(composite); - auto chain = access_chain_internal(composite, elems, length, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr); + auto chain = access_chain_internal(composite, elems, length, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, + nullptr, nullptr); statement(chain, " = ", to_unpacked_expression(obj), ";"); set(id, to_expression(composite), result_type, true); invalid_expressions.insert(composite); @@ -12523,7 +13751,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) set(id, to_name(id), result_type, true); } - auto chain = access_chain_internal(id, elems, length, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr); + auto chain = access_chain_internal(id, elems, length, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, nullptr, nullptr); statement(chain, " = ", to_unpacked_expression(obj), ";"); } @@ -12846,8 +14074,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) auto expr = join(enclose_expression(to_unpacked_row_major_matrix_expression(ops[3])), " * ", enclose_expression(to_unpacked_row_major_matrix_expression(ops[2]))); bool forward = should_forward(ops[2]) && should_forward(ops[3]); - auto &e = emit_op(ops[0], ops[1], expr, forward); - e.need_transpose = true; + emit_transposed_op(ops[0], ops[1], expr, forward); a->need_transpose = true; b->need_transpose = true; inherit_expression_dependencies(ops[1], ops[2]); @@ -12870,8 +14097,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) auto expr = join(enclose_expression(to_unpacked_row_major_matrix_expression(ops[2])), " * ", to_enclosed_unpacked_expression(ops[3])); bool forward = should_forward(ops[2]) && should_forward(ops[3]); - auto &e = emit_op(ops[0], ops[1], expr, forward); - e.need_transpose = true; + emit_transposed_op(ops[0], ops[1], expr, forward); a->need_transpose = true; inherit_expression_dependencies(ops[1], ops[2]); inherit_expression_dependencies(ops[1], ops[3]); @@ -12954,13 +14180,24 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) uint32_t op0 = ops[2]; uint32_t op1 = ops[3]; - // Needs special handling. + auto &out_type = get(result_type); + bool forward = should_forward(op0) && should_forward(op1); - auto expr = join(to_enclosed_expression(op0), " - ", to_enclosed_expression(op1), " * ", "(", - to_enclosed_expression(op0), " / ", to_enclosed_expression(op1), ")"); + string cast_op0, cast_op1; + auto expected_type = binary_op_bitcast_helper(cast_op0, cast_op1, int_type, op0, op1, false); + + // Needs special handling. + auto expr = join(cast_op0, " - ", cast_op1, " * ", "(", cast_op0, " / ", cast_op1, ")"); if (implicit_integer_promotion) + { expr = join(type_to_glsl(get(result_type)), '(', expr, ')'); + } + else if (out_type.basetype != int_type) + { + expected_type.basetype = int_type; + expr = join(bitcast_glsl_op(out_type, expected_type), '(', expr, ')'); + } emit_op(result_type, result_id, expr, forward); inherit_expression_dependencies(result_id, op0); @@ -13320,13 +14557,42 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpCooperativeMatrixConvertNV: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("CooperativeMatrixConvertNV requires vulkan semantics."); + require_extension_internal("GL_NV_cooperative_matrix2"); + // fallthrough case OpFConvert: { uint32_t result_type = ops[0]; uint32_t id = ops[1]; - auto func = type_to_glsl_constructor(get(result_type)); - emit_unary_func_op(result_type, id, ops[2], func.c_str()); + auto &type = get(result_type); + + if (type.op == OpTypeCooperativeMatrixKHR && opcode == OpFConvert) + { + auto &expr_type = expression_type(ops[2]); + if (get(type.ext.cooperative.use_id).scalar() != + get(expr_type.ext.cooperative.use_id).scalar()) + { + // Somewhat questionable with spec constant uses. + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("NV_cooperative_matrix2 requires vulkan semantics."); + require_extension_internal("GL_NV_cooperative_matrix2"); + } + } + + if ((type.basetype == SPIRType::FloatE4M3 || type.basetype == SPIRType::FloatE5M2) && + has_decoration(id, DecorationSaturatedToLargestFloat8NormalConversionEXT)) + { + emit_uninitialized_temporary_expression(result_type, id); + statement("saturatedConvertEXT(", to_expression(id), ", ", to_unpacked_expression(ops[2]), ");"); + } + else + { + auto func = type_to_glsl_constructor(type); + emit_unary_func_op(result_type, id, ops[2], func.c_str()); + } break; } @@ -13545,8 +14811,11 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) const char *increment = unsigned_type ? "0u" : "0"; emit_op(ops[0], ops[1], join(op, "(", - to_non_uniform_aware_expression(ops[2]), ", ", increment, ")"), false); + to_atomic_ptr_expression(ops[2]), ", ", increment, ")"), false); flush_all_atomic_capable_variables(); + + if (type.basetype == SPIRType::UInt64 || type.basetype == SPIRType::Int64) + require_extension_internal("GL_EXT_shader_atomic_int64"); break; } @@ -13558,8 +14827,12 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) // Ignore semantics for now, probably only relevant to CL. uint32_t val = ops[3]; const char *op = check_atomic_image(ptr) ? "imageAtomicExchange" : "atomicExchange"; - statement(op, "(", to_non_uniform_aware_expression(ptr), ", ", to_expression(val), ");"); + statement(op, "(", to_atomic_ptr_expression(ptr), ", ", to_expression(val), ");"); flush_all_atomic_capable_variables(); + + auto &type = expression_type(ptr); + if (type.basetype == SPIRType::UInt64 || type.basetype == SPIRType::Int64) + require_extension_internal("GL_EXT_shader_atomic_int64"); break; } @@ -13594,7 +14867,10 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) increment = "-1"; emit_op(ops[0], ops[1], - join(op, "(", to_non_uniform_aware_expression(ops[2]), ", ", increment, ")"), false); + join(op, "(", to_atomic_ptr_expression(ops[2]), ", ", increment, ")"), false); + + if (type.basetype == SPIRType::UInt64 || type.basetype == SPIRType::Int64) + require_extension_internal("GL_EXT_shader_atomic_int64"); } flush_all_atomic_capable_variables(); @@ -13613,9 +14889,13 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) { const char *op = check_atomic_image(ops[2]) ? "imageAtomicAdd" : "atomicAdd"; forced_temporaries.insert(ops[1]); - auto expr = join(op, "(", to_non_uniform_aware_expression(ops[2]), ", -", to_enclosed_expression(ops[5]), ")"); + auto expr = join(op, "(", to_atomic_ptr_expression(ops[2]), ", -", to_enclosed_expression(ops[5]), ")"); emit_op(ops[0], ops[1], expr, should_forward(ops[2]) && should_forward(ops[5])); flush_all_atomic_capable_variables(); + + auto &type = get(ops[0]); + if (type.basetype == SPIRType::UInt64 || type.basetype == SPIRType::Int64) + require_extension_internal("GL_EXT_shader_atomic_int64"); break; } @@ -13765,7 +15045,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) op = "textureQueryLod"; auto sampler_expr = to_expression(ops[2]); - if (has_decoration(ops[2], DecorationNonUniform)) + if (is_descriptor_non_uniform(ops[2])) { if (maybe_get_backing_variable(ops[2])) convert_non_uniform_expression(sampler_expr, ops[2]); @@ -14040,23 +15320,28 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpUntypedImageTexelPointerEXT: case OpImageTexelPointer: { + bool untyped = opcode == OpUntypedImageTexelPointerEXT; uint32_t result_type = ops[0]; uint32_t id = ops[1]; - auto coord_expr = to_expression(ops[3]); - auto target_coord_type = expression_type(ops[3]); - target_coord_type.basetype = SPIRType::Int; - coord_expr = bitcast_expression(target_coord_type, expression_type(ops[3]).basetype, coord_expr); + uint32_t image_id = ops[untyped ? 3 : 2]; + uint32_t coord_id = ops[untyped ? 4 : 3]; - auto expr = join(to_expression(ops[2]), ", ", coord_expr); + auto coord_expr = to_expression(coord_id); + auto target_coord_type = expression_type(coord_id); + target_coord_type.basetype = SPIRType::Int; + coord_expr = bitcast_expression(target_coord_type, expression_type(coord_id).basetype, coord_expr); + + auto expr = join(to_expression(image_id), ", ", coord_expr); auto &e = set(id, expr, result_type, true); // When using the pointer, we need to know which variable it is actually loaded from. - auto *var = maybe_get_backing_variable(ops[2]); + auto *var = maybe_get_backing_variable(image_id); e.loaded_from = var ? var->self : ID(0); - inherit_expression_dependencies(id, ops[3]); + inherit_expression_dependencies(id, coord_id); break; } @@ -14148,6 +15433,110 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpImageSampleWeightedQCOM: + case OpImageBoxFilterQCOM: + case OpImageBlockMatchSSDQCOM: + case OpImageBlockMatchSADQCOM: + { + require_extension_internal("GL_QCOM_image_processing"); + uint32_t result_type_id = ops[0]; + uint32_t id = ops[1]; + string expr; + switch (opcode) + { + case OpImageSampleWeightedQCOM: + expr = "textureWeightedQCOM"; + break; + case OpImageBoxFilterQCOM: + expr = "textureBoxFilterQCOM"; + break; + case OpImageBlockMatchSSDQCOM: + expr = "textureBlockMatchSSDQCOM"; + break; + case OpImageBlockMatchSADQCOM: + expr = "textureBlockMatchSADQCOM"; + break; + default: + SPIRV_CROSS_THROW("Invalid opcode for QCOM_image_processing."); + } + expr += "("; + + bool forward = false; + expr += to_expression(ops[2]); + expr += ", " + to_expression(ops[3]); + + switch (opcode) + { + case OpImageSampleWeightedQCOM: + expr += ", " + to_non_uniform_aware_expression(ops[4]); + break; + case OpImageBoxFilterQCOM: + expr += ", " + to_expression(ops[4]); + break; + case OpImageBlockMatchSSDQCOM: + case OpImageBlockMatchSADQCOM: + expr += ", " + to_non_uniform_aware_expression(ops[4]); + expr += ", " + to_expression(ops[5]); + expr += ", " + to_expression(ops[6]); + break; + default: + SPIRV_CROSS_THROW("Invalid opcode for QCOM_image_processing."); + } + + expr += ")"; + emit_op(result_type_id, id, expr, forward); + + inherit_expression_dependencies(id, ops[3]); + if (opcode == OpImageBlockMatchSSDQCOM || opcode == OpImageBlockMatchSADQCOM) + inherit_expression_dependencies(id, ops[5]); + + break; + } + + case OpImageBlockMatchWindowSSDQCOM: + case OpImageBlockMatchWindowSADQCOM: + case OpImageBlockMatchGatherSSDQCOM: + case OpImageBlockMatchGatherSADQCOM: + { + require_extension_internal("GL_QCOM_image_processing2"); + uint32_t result_type_id = ops[0]; + uint32_t id = ops[1]; + string expr; + switch (opcode) + { + case OpImageBlockMatchWindowSSDQCOM: + expr = "textureBlockMatchWindowSSDQCOM"; + break; + case OpImageBlockMatchWindowSADQCOM: + expr = "textureBlockMatchWindowSADQCOM"; + break; + case OpImageBlockMatchGatherSSDQCOM: + expr = "textureBlockMatchGatherSSDQCOM"; + break; + case OpImageBlockMatchGatherSADQCOM: + expr = "textureBlockMatchGatherSADQCOM"; + break; + default: + SPIRV_CROSS_THROW("Invalid opcode for QCOM_image_processing2."); + } + expr += "("; + + bool forward = false; + expr += to_expression(ops[2]); + expr += ", " + to_expression(ops[3]); + + expr += ", " + to_non_uniform_aware_expression(ops[4]); + expr += ", " + to_expression(ops[5]); + expr += ", " + to_expression(ops[6]); + + expr += ")"; + emit_op(result_type_id, id, expr, forward); + + inherit_expression_dependencies(id, ops[3]); + inherit_expression_dependencies(id, ops[5]); + break; + } + // Compute case OpControlBarrier: case OpMemoryBarrier: @@ -14315,6 +15704,20 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpExtInstWithForwardRefsKHR: + { + uint32_t extension_set = ops[2]; + auto ext = get(extension_set).ext; + if (ext != SPIRExtension::SPV_debug_info && + ext != SPIRExtension::NonSemanticShaderDebugInfo && + ext != SPIRExtension::NonSemanticGeneric) + { + SPIRV_CROSS_THROW("Unexpected use of ExtInstWithForwardRefsKHR."); + } + + break; + } + case OpExtInst: { uint32_t extension_set = ops[2]; @@ -14340,8 +15743,11 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) { emit_spv_amd_gcn_shader_op(ops[0], ops[1], ops[3], &ops[4], length - 4); } + else if (ext == SPIRExtension::NonSemanticShaderDebugInfo) + { + emit_non_semantic_shader_debug_info(ops[0], ops[1], ops[3], &ops[4], length - 4); + } else if (ext == SPIRExtension::SPV_debug_info || - ext == SPIRExtension::NonSemanticShaderDebugInfo || ext == SPIRExtension::NonSemanticGeneric) { break; // Ignore SPIR-V debug information extended instructions. @@ -14355,7 +15761,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) SPIRV_CROSS_THROW("Debug printf is only supported in Vulkan GLSL.\n"); require_extension_internal("GL_EXT_debug_printf"); auto &format_string = get(ops[4]).str; - string expr = join("debugPrintfEXT(\"", format_string, "\""); + string expr = join(backend.printf_function, "(\"", format_string, "\""); for (uint32_t i = 5; i < length; i++) { expr += ", "; @@ -14487,7 +15893,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) uint32_t result_type = ops[0]; uint32_t id = ops[1]; - if (type.image.dim == spv::DimSubpassData) + if (type.image.dim == DimSubpassData) { emit_unary_func_op(result_type, id, ops[2], "fragmentMaskFetchAMD"); } @@ -14506,7 +15912,7 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) uint32_t result_type = ops[0]; uint32_t id = ops[1]; - if (type.image.dim == spv::DimSubpassData) + if (type.image.dim == DimSubpassData) { emit_binary_func_op(result_type, id, ops[2], ops[4], "fragmentFetchAMD"); } @@ -14554,6 +15960,9 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) case OpGroupNonUniformLogicalXor: case OpGroupNonUniformQuadSwap: case OpGroupNonUniformQuadBroadcast: + case OpGroupNonUniformQuadAllKHR: + case OpGroupNonUniformQuadAnyKHR: + case OpGroupNonUniformRotateKHR: emit_subgroup_op(instruction); break; @@ -14713,6 +16122,11 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) flush_variable_declaration(ops[0]); statement("rayQueryConfirmIntersectionEXT(", to_expression(ops[0]), ");"); break; + case OpRayQueryGetIntersectionTriangleVertexPositionsKHR: + flush_variable_declaration(ops[1]); + emit_uninitialized_temporary_expression(ops[0], ops[1]); + statement("rayQueryGetIntersectionTriangleVertexPositionsEXT(", to_expression(ops[2]), ", bool(", to_expression(ops[3]), "), ", to_expression(ops[1]), ");"); + break; #define GLSL_RAY_QUERY_GET_OP(op) \ case OpRayQueryGet##op##KHR: \ flush_variable_declaration(ops[2]); \ @@ -14743,7 +16157,81 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) GLSL_RAY_QUERY_GET_OP2(IntersectionWorldToObject); #undef GLSL_RAY_QUERY_GET_OP #undef GLSL_RAY_QUERY_GET_OP2 + case OpRayQueryGetClusterIdNV: + flush_variable_declaration(ops[2]); + emit_op(ops[0], ops[1], join("rayQueryGetIntersectionClusterIdNV(", to_expression(ops[2]), ", ", "bool(", to_expression(ops[3]), "))"), false); + break; + case OpTensorQuerySizeARM: + flush_variable_declaration(ops[1]); + // tensorSizeARM(tensor, dimension) + emit_binary_func_op(ops[0], ops[1], ops[2], ops[3], "tensorSizeARM"); + break; + case OpTensorReadARM: + { + flush_variable_declaration(ops[1]); + emit_uninitialized_temporary_expression(ops[0], ops[1]); + SmallVector args { + to_expression(ops[2]), // tensor + to_expression(ops[3]), // coordinates + to_expression(ops[1]), // out value + }; + if (length > 4) + { + std::string tensor_operands; + if (ops[4] == 0) + tensor_operands = "0x0u"; + else if (ops[4] == TensorOperandsNontemporalARMMask) + tensor_operands = "gl_TensorOperandsNonTemporalARM"; + else if (ops[4] == TensorOperandsOutOfBoundsValueARMMask) + tensor_operands = "gl_TensorOperandsOutOfBoundsValueARM"; + else if (ops[4] == (TensorOperandsNontemporalARMMask | TensorOperandsOutOfBoundsValueARMMask)) + tensor_operands = "gl_TensorOperandsNonTemporalARM | gl_TensorOperandsOutOfBoundsValueARM"; + else + SPIRV_CROSS_THROW("Invalid tensorOperands for tensorReadARM."); + if ((ops[4] & TensorOperandsOutOfBoundsValueARMMask) && length != 6) + SPIRV_CROSS_THROW("gl_TensorOperandsOutOfBoundsValueARM requires an outOfBoundsValue argument."); + args.push_back(tensor_operands); // tensorOperands + } + if (length >= 6) + { + if ((length > 6) || (ops[4] & TensorOperandsOutOfBoundsValueARMMask) == 0) + SPIRV_CROSS_THROW("Too many arguments to tensorReadARM."); + args.push_back(to_expression(ops[5])); // outOfBoundsValue + } + + // tensorRead(tensor, sizeof(type), coordinates, value, operand, ...) + statement("tensorReadARM(", merge(args), ");"); + break; + } + case OpTensorWriteARM: + { + flush_variable_declaration(ops[0]); + + SmallVector args { + to_expression(ops[0]), // tensor + to_expression(ops[1]), // coordinates + to_expression(ops[2]), // out value + }; + + if (length > 3) + { + std::string tensor_operands; + if (ops[3] == 0) + tensor_operands = "0x0u"; + else if (ops[3] == TensorOperandsNontemporalARMMask) + tensor_operands = "gl_TensorOperandsNonTemporalARM"; + else + SPIRV_CROSS_THROW("Invalid tensorOperands for tensorWriteARM."); + args.push_back(tensor_operands); // tensorOperands + } + if (length > 4) + SPIRV_CROSS_THROW("Too many arguments to tensorWriteARM."); + + // tensorWrite(tensor, sizeof(type), coordinates, value) + statement("tensorWriteARM(", merge(args), ");"); + break; + } case OpConvertUToAccelerationStructureKHR: { require_extension_internal("GL_EXT_ray_tracing"); @@ -14773,8 +16261,8 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) case OpConvertUToPtr: { auto &type = get(ops[0]); - if (type.storage != StorageClassPhysicalStorageBufferEXT) - SPIRV_CROSS_THROW("Only StorageClassPhysicalStorageBufferEXT is supported by OpConvertUToPtr."); + if (type.storage != StorageClassPhysicalStorageBuffer) + SPIRV_CROSS_THROW("Only StorageClassPhysicalStorageBuffer is supported by OpConvertUToPtr."); auto &in_type = expression_type(ops[2]); if (in_type.vecsize == 2) @@ -14789,8 +16277,8 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) { auto &type = get(ops[0]); auto &ptr_type = expression_type(ops[2]); - if (ptr_type.storage != StorageClassPhysicalStorageBufferEXT) - SPIRV_CROSS_THROW("Only StorageClassPhysicalStorageBufferEXT is supported by OpConvertPtrToU."); + if (ptr_type.storage != StorageClassPhysicalStorageBuffer) + SPIRV_CROSS_THROW("Only StorageClassPhysicalStorageBuffer is supported by OpConvertPtrToU."); if (type.vecsize == 2) require_extension_internal("GL_EXT_buffer_reference_uvec2"); @@ -14889,6 +16377,287 @@ void CompilerGLSL::emit_instruction(const Instruction &instruction) break; } + case OpCooperativeVectorLoadNV: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + + emit_uninitialized_temporary_expression(result_type, id); + + statement("coopVecLoadNV(", to_expression(id), ", ", to_expression(ops[2]), ", ", to_expression(ops[3]), ");"); + register_read(id, ops[2], false); + break; + } + + case OpCooperativeVectorStoreNV: + { + uint32_t id = ops[0]; + + statement("coopVecStoreNV(", to_expression(ops[2]), ", ", to_expression(id), ", ", to_expression(ops[1]), ");"); + register_write(ops[2]); + break; + } + + case OpCooperativeVectorOuterProductAccumulateNV: + { + auto buf = ops[0]; + auto offset = ops[1]; + auto v1 = ops[2]; + auto v2 = ops[3]; + auto matrix_layout_id = ops[4]; + auto matrix_iterpretation_id = ops[5]; + auto matrix_stride_id = length >= 6 ? ops[6] : 0; + statement(join("coopVecOuterProductAccumulateNV(", to_expression(v1), ", ", to_expression(v2), ", ", + to_expression(buf), ", ", to_expression(offset), ", ", + matrix_stride_id ? to_expression(matrix_stride_id) : "0", + ", ", to_pretty_expression_if_int_constant( + matrix_layout_id, std::begin(CoopVecMatrixLayoutNames), std::end(CoopVecMatrixLayoutNames)), + ", ", to_pretty_expression_if_int_constant( + matrix_iterpretation_id, std::begin(CoopVecComponentTypeNames), std::end(CoopVecComponentTypeNames)), + ");")); + register_write(ops[0]); + break; + } + + case OpCooperativeVectorReduceSumAccumulateNV: + { + auto buf = ops[0]; + auto offset = ops[1]; + auto v1 = ops[2]; + statement(join("coopVecReduceSumAccumulateNV(", to_expression(v1), ", ", to_expression(buf), ", ", + to_expression(offset), ");")); + register_write(ops[0]); + break; + } + + case OpCooperativeVectorMatrixMulNV: + case OpCooperativeVectorMatrixMulAddNV: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + + emit_uninitialized_temporary_expression(result_type, id); + + std::string stmt; + switch (opcode) + { + case OpCooperativeVectorMatrixMulAddNV: + stmt += "coopVecMatMulAddNV("; + break; + case OpCooperativeVectorMatrixMulNV: + stmt += "coopVecMatMulNV("; + break; + default: + SPIRV_CROSS_THROW("Invalid op code for coopvec instruction."); + } + for (uint32_t i = 1; i < length; i++) + { + // arguments 3, 6 and in case of MulAddNv also 9 use component type int constants + if (i == 3 || i == 6 || (i == 9 && opcode == OpCooperativeVectorMatrixMulAddNV)) + { + stmt += to_pretty_expression_if_int_constant( + ops[i], std::begin(CoopVecComponentTypeNames), std::end(CoopVecComponentTypeNames)); + } + else if ((i == 12 && opcode == OpCooperativeVectorMatrixMulAddNV) || + (i == 9 && opcode == OpCooperativeVectorMatrixMulNV)) + { + stmt += to_pretty_expression_if_int_constant( + ops[i], std::begin(CoopVecMatrixLayoutNames), std::end(CoopVecMatrixLayoutNames)); + } + else + stmt += to_expression(ops[i]); + + if (i < length - 1) + stmt += ", "; + } + stmt += ");"; + statement(stmt); + break; + } + + case OpCooperativeMatrixLengthKHR: + { + // Need to synthesize a dummy temporary, since the SPIR-V opcode is based on the type. + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + set( + id, join(type_to_glsl(get(result_type)), + "(", type_to_glsl(get(ops[2])), "(0).length())"), + result_type, true); + break; + } + + case OpCooperativeMatrixLoadKHR: + { + // Spec contradicts itself if stride is optional or not. + if (length < 5) + SPIRV_CROSS_THROW("Stride is not provided."); + + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + emit_uninitialized_temporary_expression(result_type, id); + + auto expr = to_expression(ops[2]); + pair split_expr; + if (!is_forcing_recompilation()) + split_expr = split_coopmat_pointer(expr); + + string layout_expr = to_pretty_expression_if_int_constant( + ops[3], std::begin(CoopMatMatrixLayoutNames), std::end(CoopMatMatrixLayoutNames)); + statement("coopMatLoad(", to_expression(id), ", ", split_expr.first, ", ", split_expr.second, ", ", + to_expression(ops[4]), ", ", layout_expr, ");"); + + register_read(id, ops[2], false); + break; + } + + case OpCooperativeMatrixStoreKHR: + { + // Spec contradicts itself if stride is optional or not. + if (length < 4) + SPIRV_CROSS_THROW("Stride is not provided."); + + // SPIR-V and GLSL don't agree how to pass the expression. + // In SPIR-V it's a pointer, but in GLSL it's reference to array + index. + + auto expr = to_expression(ops[0]); + pair split_expr; + if (!is_forcing_recompilation()) + split_expr = split_coopmat_pointer(expr); + + string layout_expr = to_pretty_expression_if_int_constant( + ops[2], std::begin(CoopMatMatrixLayoutNames), std::end(CoopMatMatrixLayoutNames)); + + statement("coopMatStore(", to_expression(ops[1]), ", ", split_expr.first, ", ", split_expr.second, ", ", + to_expression(ops[3]), ", ", layout_expr, ");"); + + // TODO: Do we care about memory operands? + + register_write(ops[0]); + break; + } + + case OpCooperativeMatrixMulAddKHR: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t A = ops[2]; + uint32_t B = ops[3]; + uint32_t C = ops[4]; + bool forward = should_forward(A) && should_forward(B) && should_forward(C); + emit_op(result_type, id, + join("coopMatMulAdd(", + to_unpacked_expression(A), ", ", + to_unpacked_expression(B), ", ", + to_unpacked_expression(C), ", ", + (length >= 6 ? ops[5] : 0), + ")"), + forward); + + inherit_expression_dependencies(id, A); + inherit_expression_dependencies(id, B); + inherit_expression_dependencies(id, C); + break; + } + + case OpCompositeConstructReplicateEXT: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + + auto &type = get(result_type); + auto value_to_replicate = to_expression(ops[2]); + std::string rhs; + // Matrices don't have a replicating constructor for vectors. Need to manually replicate + if (type.op == OpTypeMatrix || type.op == OpTypeArray) + { + if (type.op == OpTypeArray && type.array.size() != 1) + { + SPIRV_CROSS_THROW( + "Multi-dimensional arrays currently not supported for OpCompositeConstructReplicateEXT"); + } + uint32_t num_elements = type.op == OpTypeMatrix ? type.columns : type.array[0]; + if (backend.use_initializer_list && type.op == OpTypeArray) + { + rhs += "{"; + } + else + { + rhs += type_to_glsl_constructor(type); + rhs += "("; + } + for (uint32_t i = 0; i < num_elements; i++) + { + rhs += value_to_replicate; + if (i < num_elements - 1) + rhs += ", "; + } + if (backend.use_initializer_list && type.op == OpTypeArray) + rhs += "}"; + else + rhs += ")"; + } + else + { + rhs = join(type_to_glsl(type), "(", to_expression(ops[2]), ")"); + } + emit_op(result_type, id, rhs, true); + break; + } + + case OpSDot: + case OpUDot: + case OpSUDot: + case OpSDotAccSat: + case OpUDotAccSat: + case OpSUDotAccSat: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + + bool is_acc_sat = opcode == OpSDotAccSat || opcode == OpUDotAccSat || opcode == OpSUDotAccSat; + + if (length == (is_acc_sat ? 6 : 5)) + { + if (ops[length - 1] != PackedVectorFormatPackedVectorFormat4x8Bit) + SPIRV_CROSS_THROW("Only 4x8bit packing is supported."); + } + + IntegerDotProduct idot = {}; + idot.argument_type[0] = expression_type_id(ops[2]); + idot.argument_type[1] = expression_type_id(ops[3]); + idot.result_type = result_type; + idot.op = opcode; + add_integer_dot_product_polyfill(idot); + + auto expr = join(integer_dot_product_entry_point(idot), "(", to_expression(ops[2]), ", ", to_expression(ops[3])); + + if (is_acc_sat) + { + expr += ", "; + expr += to_expression(ops[4]); + } + + if (expression_type(ops[2]).vecsize == 1) + { + expr += ", "; + expr += to_string(PackedVectorFormatPackedVectorFormat4x8Bit); + } + + expr += ")"; + + bool forward = should_forward(ops[2]) && should_forward(ops[3]); + if (is_acc_sat && forward) + forward = should_forward(ops[4]); + + emit_op(result_type, id, expr, forward); + inherit_expression_dependencies(id, ops[2]); + inherit_expression_dependencies(id, ops[3]); + if (is_acc_sat) + inherit_expression_dependencies(id, ops[4]); + break; + } + default: statement("// unimplemented op ", instruction.op); break; @@ -15079,7 +16848,7 @@ string CompilerGLSL::variable_decl(const SPIRType &type, const string &name, uin { string type_name = type_to_glsl(type, id); remap_variable_type_name(type, name, type_name); - return join(type_name, " ", name, type_to_array_glsl(type)); + return join(type_name, " ", name, type_to_array_glsl(type, id)); } bool CompilerGLSL::variable_decl_is_remapped_storage(const SPIRVariable &var, StorageClass storage) const @@ -15106,15 +16875,11 @@ void CompilerGLSL::emit_struct_member(const SPIRType &type, uint32_t member_type if (is_block) qualifiers = to_interpolation_qualifiers(memberflags); - statement(layout_for_member(type, index), qualifiers, qualifier, flags_to_qualifiers_glsl(membertype, memberflags), + statement(layout_for_member(type, index), qualifiers, qualifier, flags_to_qualifiers_glsl(membertype, 0, memberflags), variable_decl(membertype, to_member_name(type, index)), ";"); } -void CompilerGLSL::emit_struct_padding_target(const SPIRType &) -{ -} - -string CompilerGLSL::flags_to_qualifiers_glsl(const SPIRType &type, const Bitset &flags) +string CompilerGLSL::flags_to_qualifiers_glsl(const SPIRType &type, uint32_t id, const Bitset &flags) { // GL_EXT_buffer_reference variables can be marked as restrict. if (flags.get(DecorationRestrictPointerEXT)) @@ -15122,8 +16887,12 @@ string CompilerGLSL::flags_to_qualifiers_glsl(const SPIRType &type, const Bitset string qual; - if (type_is_floating_point(type) && flags.get(DecorationNoContraction) && backend.support_precise_qualifier) + if (type_is_floating_point(type) && + (flags.get(DecorationNoContraction) || (type.self && has_legacy_nocontract(type.self, id))) && + backend.support_precise_qualifier) + { qual = "precise "; + } // Structs do not have precision qualifiers, neither do doubles (desktop only anyways, so no mediump/highp). bool type_supports_precision = @@ -15192,7 +16961,7 @@ string CompilerGLSL::to_precision_qualifiers_glsl(uint32_t id) if (result_type.width < 32) return "mediump "; } - return flags_to_qualifiers_glsl(type, ir.meta[id].decoration.decoration_flags); + return flags_to_qualifiers_glsl(type, id, ir.meta[id].decoration.decoration_flags); } void CompilerGLSL::fixup_io_block_patch_primitive_qualifiers(const SPIRVariable &var) @@ -15275,6 +17044,13 @@ string CompilerGLSL::to_qualifiers_glsl(uint32_t id) SPIRV_CROSS_THROW("Cannot use GL_EXT_shader_image_load_formatted in ESSL."); } } + else if (type.basetype == SPIRType::Tensor) + { + if (flags.get(DecorationNonWritable)) + res += "readonly "; + if (flags.get(DecorationNonReadable)) + res += "writeonly "; + } res += to_precision_qualifiers_glsl(id); @@ -15287,9 +17063,21 @@ string CompilerGLSL::argument_decl(const SPIRFunction::Parameter &arg) auto &type = expression_type(arg.id); const char *direction = ""; - if (type.pointer) + if (is_pointer(type) && + (type.storage == StorageClassFunction || + type.storage == StorageClassPrivate || + type.storage == StorageClassOutput)) { - if (arg.write_count && arg.read_count) + // If we're passing around block types to function, we really mean reference in a pointer sense, + // but DXC does not like inout for mesh blocks, so workaround that. out is technically not correct, + // but it works in practice due to legalization. It's ... not great, but you gotta do what you gotta do. + // GLSL will never hit this case since it's not valid. + if (type.storage == StorageClassOutput && get_execution_model() == ExecutionModelMeshEXT && + has_decoration(type.self, DecorationBlock) && is_builtin_type(type) && arg.write_count) + { + direction = "out "; + } + else if (arg.write_count && arg.read_count) direction = "inout "; else if (arg.write_count) direction = "out "; @@ -15352,13 +17140,24 @@ string CompilerGLSL::variable_decl(const SPIRVariable &variable) else if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) res += join(" = ", to_zero_initialized_expression(get_variable_data_type_id(variable))); } - else if (variable.initializer && !variable_decl_is_remapped_storage(variable, StorageClassWorkgroup)) + else if (variable.initializer) { - uint32_t expr = variable.initializer; - if (ir.ids[expr].get_type() != TypeUndef) - res += join(" = ", to_initializer_expression(variable)); - else if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) - res += join(" = ", to_zero_initialized_expression(get_variable_data_type_id(variable))); + if (!variable_decl_is_remapped_storage(variable, StorageClassWorkgroup)) + { + uint32_t expr = variable.initializer; + if (ir.ids[expr].get_type() != TypeUndef) + res += join(" = ", to_initializer_expression(variable)); + else if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) + res += join(" = ", to_zero_initialized_expression(get_variable_data_type_id(variable))); + } + else + { + // Workgroup memory requires special handling. First, it can only be Null-Initialized. + // GLSL will handle this with null initializer, while others require more work after the decl + require_extension_internal("GL_EXT_null_initializer"); + if (!backend.constant_null_initializer.empty()) + res += join(" = ", backend.constant_null_initializer); + } } return res; @@ -15436,9 +17235,9 @@ string CompilerGLSL::to_array_size(const SPIRType &type, uint32_t index) return ""; } -string CompilerGLSL::type_to_array_glsl(const SPIRType &type) +string CompilerGLSL::type_to_array_glsl(const SPIRType &type, uint32_t) { - if (type.pointer && type.storage == StorageClassPhysicalStorageBufferEXT && type.basetype != SPIRType::Struct) + if (type.pointer && type.storage == StorageClassPhysicalStorageBuffer && type.basetype != SPIRType::Struct) { // We are using a wrapped pointer type, and we should not emit any array declarations here. return ""; @@ -15483,7 +17282,7 @@ string CompilerGLSL::type_to_array_glsl(const SPIRType &type) } } -string CompilerGLSL::image_type_glsl(const SPIRType &type, uint32_t id) +string CompilerGLSL::image_type_glsl(const SPIRType &type, uint32_t id, bool /*member*/) { auto &imagetype = get(type.image.type); string res; @@ -15566,7 +17365,7 @@ string CompilerGLSL::image_type_glsl(const SPIRType &type, uint32_t id) case DimBuffer: if (options.es && options.version < 320) require_extension_internal("GL_EXT_texture_buffer"); - else if (!options.es && options.version < 300) + else if (!options.es && options.version < 140) require_extension_internal("GL_EXT_texture_buffer_object"); res += "Buffer"; break; @@ -15635,17 +17434,29 @@ string CompilerGLSL::type_to_glsl_constructor(const SPIRType &type) // depend on a specific object's use of that type. string CompilerGLSL::type_to_glsl(const SPIRType &type, uint32_t id) { - if (type.pointer && type.storage == StorageClassPhysicalStorageBufferEXT && type.basetype != SPIRType::Struct) + if (is_physical_pointer(type) && !is_physical_pointer_to_buffer_block(type)) { // Need to create a magic type name which compacts the entire type information. - string name = type_to_glsl(get_pointee_type(type)); - for (size_t i = 0; i < type.array.size(); i++) + auto *parent = &get_pointee_type(type); + string name = type_to_glsl(*parent); + + uint32_t array_stride = get_decoration(type.parent_type, DecorationArrayStride); + + // Resolve all array dimensions in one go since once we lose the pointer type, + // array information is left to to_array_type_glsl. The base type loses array information. + while (is_array(*parent)) { - if (type.array_size_literal[i]) - name += join(type.array[i], "_"); + if (parent->array_size_literal.back()) + name += join(type.array.back(), "_"); else - name += join("id", type.array[i], "_"); + name += join("id", type.array.back(), "_"); + + name += "stride_" + std::to_string(array_stride); + + array_stride = get_decoration(parent->parent_type, DecorationArrayStride); + parent = &get(parent->parent_type); } + name += "Pointer"; return name; } @@ -15674,6 +17485,14 @@ string CompilerGLSL::type_to_glsl(const SPIRType &type, uint32_t id) case SPIRType::RayQuery: return "rayQueryEXT"; + case SPIRType::Tensor: + if (type.ext.tensor.rank == 0) + SPIRV_CROSS_THROW("GLSL tensors must have a Rank."); + if (type.ext.tensor.shape != 0) + SPIRV_CROSS_THROW("GLSL tensors cannot have a Shape."); + return join("tensorARM<", type_to_glsl(get(type.ext.tensor.type)), ", ", + to_expression(type.ext.tensor.rank), ">"); + case SPIRType::Void: return "void"; @@ -15701,6 +17520,72 @@ string CompilerGLSL::type_to_glsl(const SPIRType &type, uint32_t id) require_extension_internal("GL_ARB_shader_atomic_counters"); } + if (type.op == OpTypeCooperativeVectorNV) + { + require_extension_internal("GL_NV_cooperative_vector"); + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("Cooperative vector NV only available in Vulkan."); + + std::string component_type_str = type_to_glsl(get(type.ext.coopVecNV.component_type_id)); + + return join("coopvecNV<", component_type_str, ", ", to_expression(type.ext.coopVecNV.component_count_id), ">"); + } + + const SPIRType *coop_type = &type; + while (is_pointer(*coop_type) || is_array(*coop_type)) + coop_type = &get(coop_type->parent_type); + + if (coop_type->op == OpTypeCooperativeMatrixKHR) + { + require_extension_internal("GL_KHR_cooperative_matrix"); + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("Cooperative matrix only available in Vulkan."); + // GLSL doesn't support this as spec constant, which makes sense ... + uint32_t use_type = get(coop_type->ext.cooperative.use_id).scalar(); + + const char *use = nullptr; + switch (use_type) + { + case CooperativeMatrixUseMatrixAKHR: + use = "gl_MatrixUseA"; + break; + + case CooperativeMatrixUseMatrixBKHR: + use = "gl_MatrixUseB"; + break; + + case CooperativeMatrixUseMatrixAccumulatorKHR: + use = "gl_MatrixUseAccumulator"; + break; + + default: + SPIRV_CROSS_THROW("Invalid matrix use."); + } + + string scope_expr; + if (const auto *scope = maybe_get(coop_type->ext.cooperative.scope_id)) + { + if (!scope->specialization) + { + require_extension_internal("GL_KHR_memory_scope_semantics"); + if (scope->scalar() == ScopeSubgroup) + scope_expr = "gl_ScopeSubgroup"; + else if (scope->scalar() == ScopeWorkgroup) + scope_expr = "gl_ScopeWorkgroup"; + else + SPIRV_CROSS_THROW("Invalid scope for cooperative matrix."); + } + } + + if (scope_expr.empty()) + scope_expr = to_expression(coop_type->ext.cooperative.scope_id); + + return join("coopmat<", type_to_glsl(get(coop_type->parent_type)), ", ", + scope_expr, ", ", + to_expression(coop_type->ext.cooperative.rows_id), ", ", + to_expression(coop_type->ext.cooperative.columns_id), ", ", use, ">"); + } + if (type.vecsize == 1 && type.columns == 1) // Scalar builtin { switch (type.basetype) @@ -15723,6 +17608,21 @@ string CompilerGLSL::type_to_glsl(const SPIRType &type, uint32_t id) return "atomic_uint"; case SPIRType::Half: return "float16_t"; + case SPIRType::BFloat16: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("bfloat16 requires Vulkan semantics."); + require_extension_internal("GL_EXT_bfloat16"); + return "bfloat16_t"; + case SPIRType::FloatE4M3: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("floate4m3_t requires Vulkan semantics."); + require_extension_internal("GL_EXT_float_e4m3"); + return "floate4m3_t"; + case SPIRType::FloatE5M2: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("floate5m2_t requires Vulkan semantics."); + require_extension_internal("GL_EXT_float_e5m2"); + return "floate5m2_t"; case SPIRType::Float: return "float"; case SPIRType::Double: @@ -15755,6 +17655,21 @@ string CompilerGLSL::type_to_glsl(const SPIRType &type, uint32_t id) return join("uvec", type.vecsize); case SPIRType::Half: return join("f16vec", type.vecsize); + case SPIRType::BFloat16: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("bfloat16 requires Vulkan semantics."); + require_extension_internal("GL_EXT_bfloat16"); + return join("bf16vec", type.vecsize); + case SPIRType::FloatE4M3: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("floate4m3_t requires Vulkan semantics."); + require_extension_internal("GL_EXT_float_e4m3"); + return join("fe4m3vec", type.vecsize); + case SPIRType::FloatE5M2: + if (!options.vulkan_semantics) + SPIRV_CROSS_THROW("floate5m2_t requires Vulkan semantics."); + require_extension_internal("GL_EXT_float_e5m2"); + return join("fe5m2vec", type.vecsize); case SPIRType::Float: return join("vec", type.vecsize); case SPIRType::Double: @@ -15887,7 +17802,7 @@ void CompilerGLSL::flatten_buffer_block(VariableID id) flattened_buffer_blocks.insert(id); } -bool CompilerGLSL::builtin_translates_to_nonarray(spv::BuiltIn /*builtin*/) const +bool CompilerGLSL::builtin_translates_to_nonarray(BuiltIn /*builtin*/) const { return false; // GLSL itself does not need to translate array builtin types to non-array builtin types } @@ -15930,6 +17845,11 @@ void CompilerGLSL::add_function_overload(const SPIRFunction &func) // but that will not change the signature in GLSL/HLSL, // so strip the pointer type before hashing. uint32_t type_id = get_pointee_type_id(arg.type); + + // Workaround glslang bug. It seems to only consider the base type when resolving overloads. + if (get(type_id).op == OpTypeCooperativeMatrixKHR) + type_id = get(type_id).parent_type; + auto &type = get(type_id); if (!combined_image_samplers.empty()) @@ -15977,7 +17897,12 @@ void CompilerGLSL::add_function_overload(const SPIRFunction &func) void CompilerGLSL::emit_function_prototype(SPIRFunction &func, const Bitset &return_flags) { - if (func.self != ir.default_entry_point) + // In library mode default_entry_point points at the first exported + // function; treat every export as a normal function rather than as the + // shader's entry point. + const bool is_entry_point = !ir.is_library_module && func.self == ir.default_entry_point; + + if (!is_entry_point) add_function_overload(func); // Avoid shadow declarations. @@ -15986,17 +17911,19 @@ void CompilerGLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret string decl; auto &type = get(func.return_type); - decl += flags_to_qualifiers_glsl(type, return_flags); + decl += flags_to_qualifiers_glsl(type, 0, return_flags); decl += type_to_glsl(type); - decl += type_to_array_glsl(type); + decl += type_to_array_glsl(type, 0); decl += " "; - if (func.self == ir.default_entry_point) + if (is_entry_point) { // If we need complex fallback in GLSL, we just wrap main() in a function // and interlock the entire shader ... if (interlocked_is_complex) decl += "spvMainInterlockedBody"; + else if (options.use_entry_point_name) + decl += get_entry_point().name; else decl += "main"; @@ -16069,6 +17996,7 @@ void CompilerGLSL::emit_function(SPIRFunction &func, const Bitset &return_flags) { // Recursively emit functions which are called. uint32_t id = ops[2]; + emit_function(get(id), ir.meta[ops[1]].decoration.decoration_flags); } } @@ -16097,6 +18025,8 @@ void CompilerGLSL::emit_function(SPIRFunction &func, const Bitset &return_flags) { auto &var = get(v); var.deferred_declaration = false; + if (var.storage == StorageClassTaskPayloadWorkgroupEXT) + continue; if (variable_decl_is_remapped_storage(var, StorageClassWorkgroup)) { @@ -16105,6 +18035,12 @@ void CompilerGLSL::emit_function(SPIRFunction &func, const Bitset &return_flags) // Comes from MSL which can push global variables as local variables in main function. add_local_variable_name(var.self); statement(variable_decl(var), ";"); + + // "Real" workgroup variables in compute shaders needs extra caretaking. + // They need to be initialized with an extra routine as they come in arbitrary form. + if (var.storage == StorageClassWorkgroup && var.initializer) + emit_workgroup_initialization(var); + var.deferred_declaration = false; } else if (var.storage == StorageClassPrivate) @@ -16211,6 +18147,10 @@ void CompilerGLSL::emit_fixup() } } +void CompilerGLSL::emit_workgroup_initialization(const SPIRVariable &) +{ +} + void CompilerGLSL::flush_phi(BlockID from, BlockID to) { auto &child = get(to); @@ -16514,13 +18454,30 @@ string CompilerGLSL::emit_continue_block(uint32_t continue_block, bool follow_tr return merge(statements); } +// Loop variable with OpUndef init: zero-init instead of leaving uninitialized (FXC X4555/X4000). +std::string CompilerGLSL::undef_loop_variable_initializer_suffix(const SPIRVariable &var) +{ + if (!backend.requires_phi_undef_zero_init) + return ""; + + uint32_t expr = var.static_expression; + if (expr == 0 || ir.ids[expr].get_type() != TypeUndef) + return ""; + + auto &type = get(var.basetype); + if (!type_can_zero_initialize(type)) + return ""; + + return join(" = ", to_zero_initialized_expression(var.basetype)); +} + void CompilerGLSL::emit_while_loop_initializers(const SPIRBlock &block) { // While loops do not take initializers, so declare all of them outside. for (auto &loop_var : block.loop_variables) { auto &var = get(loop_var); - statement(variable_decl(var), ";"); + statement(variable_decl(var), undef_loop_variable_initializer_suffix(var), ";"); } } @@ -16552,7 +18509,10 @@ string CompilerGLSL::emit_for_loop_initializers(const SPIRBlock &block) else if (!same_types || missing_initializers == uint32_t(block.loop_variables.size())) { for (auto &loop_var : block.loop_variables) - statement(variable_decl(get(loop_var)), ";"); + { + auto &var = get(loop_var); + statement(variable_decl(var), undef_loop_variable_initializer_suffix(var), ";"); + } return ""; } else @@ -16563,10 +18523,11 @@ string CompilerGLSL::emit_for_loop_initializers(const SPIRBlock &block) for (auto &loop_var : block.loop_variables) { - uint32_t static_expr = get(loop_var).static_expression; + auto &var_for_undef = get(loop_var); + uint32_t static_expr = var_for_undef.static_expression; if (static_expr == 0 || ir.ids[static_expr].get_type() == TypeUndef) { - statement(variable_decl(get(loop_var)), ";"); + statement(variable_decl(var_for_undef), undef_loop_variable_initializer_suffix(var_for_undef), ";"); } else { @@ -16648,8 +18609,11 @@ bool CompilerGLSL::attempt_emit_loop_header(SPIRBlock &block, SPIRBlock::Method bool condition_is_temporary = forced_temporaries.find(block.condition) == end(forced_temporaries); + bool flushes_phi = flush_phi_required(block.self, block.true_block) || + flush_phi_required(block.self, block.false_block); + // This can work! We only did trivial things which could be forwarded in block body! - if (current_count == statement_count && condition_is_temporary) + if (!flushes_phi && current_count == statement_count && condition_is_temporary) { switch (continue_type) { @@ -16728,7 +18692,10 @@ bool CompilerGLSL::attempt_emit_loop_header(SPIRBlock &block, SPIRBlock::Method bool condition_is_temporary = forced_temporaries.find(child.condition) == end(forced_temporaries); - if (current_count == statement_count && condition_is_temporary) + bool flushes_phi = flush_phi_required(child.self, child.true_block) || + flush_phi_required(child.self, child.false_block); + + if (!flushes_phi && current_count == statement_count && condition_is_temporary) { uint32_t target_block = child.true_block; @@ -16814,7 +18781,11 @@ void CompilerGLSL::emit_hoisted_temporaries(SmallVector> &tempo // There are some rare scenarios where we are asked to declare pointer types as hoisted temporaries. // This should be ignored unless we're doing actual variable pointers and backend supports it. // Access chains cannot normally be lowered to temporaries in GLSL and HLSL. - if (type.pointer && !backend.native_pointers) + if (type.pointer && (!backend.native_pointers || type_is_opaque_value(get_pointee_type(type)))) + continue; + + // Anything involving opaque objects cannot be lowered to temporaries ever. + if (type_is_opaque_value(type)) continue; add_local_variable_name(tmp.second); @@ -16825,7 +18796,7 @@ void CompilerGLSL::emit_hoisted_temporaries(SmallVector> &tempo if (options.force_zero_initialized_variables && type_can_zero_initialize(type)) initializer = join(" = ", to_zero_initialized_expression(tmp.first)); - statement(flags_to_qualifiers_glsl(type, flags), variable_decl(type, to_name(tmp.second)), initializer, ";"); + statement(flags_to_qualifiers_glsl(type, tmp.second, flags), variable_decl(type, to_name(tmp.second)), initializer, ";"); hoisted_temporaries.insert(tmp.second); forced_temporaries.insert(tmp.second); @@ -16840,7 +18811,7 @@ void CompilerGLSL::emit_hoisted_temporaries(SmallVector> &tempo { uint32_t mirror_id = mirrored_precision_itr->second; auto &mirror_flags = get_decoration_bitset(mirror_id); - statement(flags_to_qualifiers_glsl(type, mirror_flags), + statement(flags_to_qualifiers_glsl(type, mirror_id, mirror_flags), variable_decl(type, to_name(mirror_id)), initializer, ";"); // The temporary might be read from before it's assigned, set up the expression now. @@ -16851,6 +18822,26 @@ void CompilerGLSL::emit_hoisted_temporaries(SmallVector> &tempo } void CompilerGLSL::emit_block_chain(SPIRBlock &block) +{ + SmallVector cleanup_stack; + BlockID next_block = emit_block_chain_inner(block); + + while (next_block != 0) + { + cleanup_stack.push_back(next_block); + next_block = emit_block_chain_inner(get(next_block)); + } + + while (!cleanup_stack.empty()) + { + emit_block_chain_cleanup(get(cleanup_stack.back())); + cleanup_stack.pop_back(); + } + + emit_block_chain_cleanup(block); +} + +BlockID CompilerGLSL::emit_block_chain_inner(SPIRBlock &block) { bool select_branch_to_true_block = false; bool select_branch_to_false_block = false; @@ -16901,12 +18892,13 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) } // Remember deferred declaration state. We will restore it before returning. - SmallVector rearm_dominated_variables(block.dominated_variables.size()); + assert(block.rearm_dominated_variables.empty()); + block.rearm_dominated_variables.resize(block.dominated_variables.size()); for (size_t i = 0; i < block.dominated_variables.size(); i++) { uint32_t var_id = block.dominated_variables[i]; auto &var = get(var_id); - rearm_dominated_variables[i] = var.deferred_declaration; + block.rearm_dominated_variables[i] = var.deferred_declaration; } // This is the method often used by spirv-opt to implement loops. @@ -17257,7 +19249,7 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) if (!collapsed_switch) { - if (block_like_switch || is_legacy_es()) + if (block_like_switch || is_legacy()) { // ESSL 1.0 is not guaranteed to support do/while. if (is_legacy_es()) @@ -17287,7 +19279,7 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) // Default case. if (!block_like_switch) { - if (is_legacy_es()) + if (is_legacy()) statement("else"); else statement("default:"); @@ -17295,7 +19287,7 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) } else { - if (is_legacy_es()) + if (is_legacy()) { statement((i ? "else " : ""), "if (", to_legacy_case_label(block.condition, literals, label_suffix), ")"); @@ -17347,7 +19339,7 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) if (block.default_block == block.next_block) { - if (is_legacy_es()) + if (is_legacy()) statement("else"); else statement("default:"); @@ -17361,7 +19353,7 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) if (!collapsed_switch) { - if (block_like_switch && !is_legacy_es()) + if ((block_like_switch || is_legacy()) && !is_legacy_es()) end_scope_decl("while(false)"); else end_scope(); @@ -17438,6 +19430,14 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) case SPIRBlock::Unreachable: { + // If the entry point ends with unreachable and has a return value, insert a return + // statement to avoid potential compiler errors from non-void functions without a return value. + if (block.return_value) + { + statement("return ", to_unpacked_expression(block.return_value), ";"); + break; + } + // Avoid emitting false fallthrough, which can happen for // if (cond) break; else discard; inside a case label. // Discard is not always implementable as a terminator. @@ -17491,6 +19491,8 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) SPIRV_CROSS_THROW("Unimplemented block terminator."); } + BlockID trailing_block_id = 0; + if (block.next_block && emit_next_block) { // If we hit this case, we're dealing with an unconditional branch, which means we will output @@ -17498,8 +19500,10 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) if (block.merge != SPIRBlock::MergeSelection) { flush_phi(block.self, block.next_block); + // For a direct branch, need to remember to invalidate expressions in the next linear block instead. - get(block.next_block).invalidate_expressions = block.invalidate_expressions; + get(block.next_block).invalidate_expressions.clear(); + std::swap(get(block.next_block).invalidate_expressions, block.invalidate_expressions); } // For switch fallthrough cases, we terminate the chain here, but we still need to handle Phi. @@ -17520,7 +19524,15 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) branch_to_continue(block.self, block.next_block); } else if (BlockID(block.self) != block.next_block) - emit_block_chain(get(block.next_block)); + { + // Recursing here is quite scary since it's quite easy to stack overflow if + // the SPIR-V is constructed a particular way. + // We have to simulate the tail call ourselves. + if (block.merge != SPIRBlock::MergeLoop) + trailing_block_id = block.next_block; + else + emit_block_chain(get(block.next_block)); + } } } @@ -17562,20 +19574,28 @@ void CompilerGLSL::emit_block_chain(SPIRBlock &block) if (is_continue(block.merge_block)) branch_to_continue(block.self, block.merge_block); else - emit_block_chain(get(block.merge_block)); + trailing_block_id = block.merge_block; } + return trailing_block_id; +} + +void CompilerGLSL::emit_block_chain_cleanup(SPIRBlock &block) +{ // Forget about control dependent expressions now. block.invalidate_expressions.clear(); - // After we return, we must be out of scope, so if we somehow have to re-emit this function, + // After we return, we must be out of scope, so if we somehow have to re-emit this block, // re-declare variables if necessary. - assert(rearm_dominated_variables.size() == block.dominated_variables.size()); + // We only need one array here for rearm_dominated_variables, + // since it should be impossible for the same block to be remitted in the same chain twice. + assert(block.rearm_dominated_variables.size() == block.dominated_variables.size()); for (size_t i = 0; i < block.dominated_variables.size(); i++) { uint32_t var = block.dominated_variables[i]; - get(var).deferred_declaration = rearm_dominated_variables[i]; + get(var).deferred_declaration = block.rearm_dominated_variables[i]; } + block.rearm_dominated_variables.clear(); // Just like for deferred declaration, we need to forget about loop variable enable // if our block chain is reinstantiated later. @@ -17887,16 +19907,28 @@ void CompilerGLSL::convert_non_uniform_expression(string &expr, uint32_t ptr_id) return; auto *var = maybe_get_backing_variable(ptr_id); - if (!var) + auto *buffer_pointer = maybe_get_backing_buffer_pointer(ptr_id); + if (!var && !buffer_pointer) return; - if (var->storage != StorageClassUniformConstant && + if (!buffer_pointer && + var->storage != StorageClassUniformConstant && var->storage != StorageClassStorageBuffer && var->storage != StorageClassUniform) return; - auto &backing_type = get(var->basetype); - if (backing_type.array.empty()) + auto &backing_type = get(var ? var->basetype : buffer_pointer->expression_type); + + bool descriptor_heap = false; + if (var) + { + auto builtin = BuiltIn(get_decoration(var->self, DecorationBuiltIn)); + descriptor_heap = builtin == BuiltInResourceHeapEXT || builtin == BuiltInSamplerHeapEXT; + } + else if (buffer_pointer) + descriptor_heap = true; + + if (!descriptor_heap && backing_type.array.empty()) return; // If we get here, we know we're accessing an arrayed resource which @@ -18117,6 +20149,34 @@ void CompilerGLSL::emit_line_directive(uint32_t file_id, uint32_t line_literal) } } +void CompilerGLSL::emit_non_semantic_shader_debug_info(uint32_t, uint32_t result_id, uint32_t eop, + const uint32_t *args, uint32_t) +{ + if (!options.emit_line_directives) + return; + + switch (eop) + { + case SPIRExtension::DebugLine: + { + // We're missing line end and columns here, but I don't think we can emit those in any meaningful way. + emit_line_directive(args[0], get(args[1]).scalar()); + break; + } + + case SPIRExtension::DebugSource: + { + // Forward the string declaration here. We ignore the optional text operand. + auto &str = get(args[0]).str; + set(result_id, str); + break; + } + + default: + break; + } +} + void CompilerGLSL::emit_copy_logical_type(uint32_t lhs_id, uint32_t lhs_type_id, uint32_t rhs_id, uint32_t rhs_type_id, SmallVector chain) { @@ -18157,9 +20217,9 @@ void CompilerGLSL::emit_copy_logical_type(uint32_t lhs_id, uint32_t lhs_type_id, AccessChainMeta lhs_meta, rhs_meta; auto lhs = access_chain_internal(lhs_id, chain.data(), uint32_t(chain.size()), - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &lhs_meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &lhs_meta, nullptr); auto rhs = access_chain_internal(rhs_id, chain.data(), uint32_t(chain.size()), - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &rhs_meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT, &rhs_meta, nullptr); uint32_t id = ir.increase_bound_by(2); lhs_id = id; @@ -18652,7 +20712,7 @@ bool CompilerGLSL::is_stage_output_location_masked(uint32_t location, uint32_t c return masked_output_locations.count({ location, component }) != 0; } -bool CompilerGLSL::is_stage_output_builtin_masked(spv::BuiltIn builtin) const +bool CompilerGLSL::is_stage_output_builtin_masked(BuiltIn builtin) const { return masked_output_builtins.count(builtin) != 0; } @@ -18756,3 +20816,223 @@ std::string CompilerGLSL::format_double(double value) const return convert_to_string(value, current_locale_radix_character); } +std::string CompilerGLSL::to_pretty_expression_if_int_constant( + uint32_t id, + const GlslConstantNameMapping *mapping_start, const GlslConstantNameMapping *mapping_end, + bool register_expression_read) +{ + auto *c = maybe_get(id); + if (c && !c->specialization) + { + auto value = c->scalar(); + auto pretty_name = std::find_if(mapping_start, mapping_end, + [value](const GlslConstantNameMapping &mapping) { return mapping.value == value; }); + if (pretty_name != mapping_end) + return pretty_name->alias; + } + return join("int(", to_expression(id, register_expression_read), ")"); +} + +uint32_t CompilerGLSL::get_fp_fast_math_flags_for_op(uint32_t result_type, uint32_t id) const +{ + uint32_t fp_flags = ~0; + + if (!type_is_floating_point(get(result_type))) + return fp_flags; + + auto &ep = get_entry_point(); + + // Per-operation flag supersedes all defaults. + if (id != 0 && has_decoration(id, DecorationFPFastMathMode)) + return get_decoration(id, DecorationFPFastMathMode); + + // Handle float_controls1 execution modes. + uint32_t width = get(result_type).width; + + bool szinp = false; + + switch (width) + { + case 8: + szinp = ep.signed_zero_inf_nan_preserve_8; + break; + + case 16: + szinp = ep.signed_zero_inf_nan_preserve_16; + break; + + case 32: + szinp = ep.signed_zero_inf_nan_preserve_32; + break; + + case 64: + szinp = ep.signed_zero_inf_nan_preserve_64; + break; + + default: + break; + } + + if (szinp) + fp_flags &= ~(FPFastMathModeNSZMask | FPFastMathModeNotInfMask | FPFastMathModeNotNaNMask); + + // Legacy NoContraction deals with any kind of transform to the expression. + if (id != 0 && has_decoration(id, DecorationNoContraction)) + fp_flags &= ~(FPFastMathModeAllowContractMask | FPFastMathModeAllowTransformMask | FPFastMathModeAllowReassocMask); + + // Handle float_controls2 execution modes. + bool found_default = false; + for (auto &fp_pair : ep.fp_fast_math_defaults) + { + if (get(fp_pair.first).width == width && fp_pair.second) + { + fp_flags &= get(fp_pair.second).scalar(); + found_default = true; + } + } + + // From SPV_KHR_float_controls2: + // "This definition implies that, if the entry point set any FPFastMathDefault execution mode + // then any type for which a default is not set uses no fast math flags + // (although this can still be overridden on a per-operation basis). + // Modules must not mix setting fast math modes explicitly using this extension and relying on older API defaults." + if (!found_default && !ep.fp_fast_math_defaults.empty()) + fp_flags = 0; + + return fp_flags; +} + +bool CompilerGLSL::has_legacy_nocontract(uint32_t result_type, uint32_t id) const +{ + const auto fp_flags = FPFastMathModeAllowContractMask | + FPFastMathModeAllowTransformMask | + FPFastMathModeAllowReassocMask; + return (get_fp_fast_math_flags_for_op(result_type, id) & fp_flags) != fp_flags; +} + +void CompilerGLSL::remap_descriptor_heap(ResourceType type, uint32_t desc_set, uint32_t binding, Dim dim) +{ + for (auto &mapping : descriptor_heap_mappings) + { + if (mapping.type == type) + { + mapping.desc_set = desc_set; + mapping.binding = binding; + mapping.dim = dim; + return; + } + } + + descriptor_heap_mappings.push_back({ type, desc_set, binding, dim }); +} + +bool CompilerGLSL::is_descriptor_non_uniform(uint32_t id) const +{ + if (has_decoration(id, DecorationNonUniform)) + return true; + + // Only infer nonuniform for descriptors. + auto &type = expression_type(id); + + if (is_pointer(type)) + { + if (type.storage != StorageClassUniform && type.storage != StorageClassStorageBuffer && + type.storage != StorageClassUniformConstant && type.storage != StorageClassImage) + return false; + } + else if (!type_is_opaque_value(type)) + { + return false; + } + + if (descriptor_heap_mappings.empty()) + return false; + + if (has_decoration(id, DecorationUniform)) + return false; + + if (std::find(ir.declared_capabilities.begin(), ir.declared_capabilities.end(), + CapabilityDescriptorHeapEXT) == ir.declared_capabilities.end()) + return false; + + // Definitely not. + if (maybe_get(id) || maybe_get(id)) + return false; + + // DescriptorHeapEXT requires that nonuniformEXT is implied, + // but if we're remapping to legacy set/binding model, glslang will not emit the cap in cross compiled source, + // so we have to enforce it. We don't have compiler-infra to deduce subgroup uniformity statically, + // so just slap it on everything. Compilers generally figure this stuff out. + return true; +} + +std::string CompilerGLSL::to_descriptor_heap_layout(const SPIRType &type, StorageClass storage) const +{ + auto resource = ResourceTypeUnknown; + Dim dim = DimMax; + + switch (type.basetype) + { + case SPIRType::Sampler: + resource = ResourceTypeSeparateSamplers; + break; + + case SPIRType::Image: + dim = type.image.dim == DimBuffer ? DimBuffer : Dim2D; + resource = type.image.sampled == 2 ? ResourceTypeStorageImage : ResourceTypeSeparateImage; + break; + + case SPIRType::SampledImage: + resource = ResourceTypeSampledImage; + break; + + case SPIRType::AccelerationStructure: + resource = ResourceTypeAccelerationStructure; + break; + + case SPIRType::AtomicCounter: + resource = ResourceTypeAtomicCounter; + break; + + case SPIRType::Struct: + { + bool ssbo = storage == StorageClassStorageBuffer || has_decoration(type.self, DecorationBufferBlock); + resource = ssbo ? ResourceTypeStorageBuffer : ResourceTypeUniformBuffer; + break; + } + + default: + break; + } + + for (auto &mapping : descriptor_heap_mappings) + { + if (mapping.type == resource) + { + bool has_match = false; + + if (type.basetype == SPIRType::Image) + { + if (dim == DimBuffer && (mapping.dim == DimMax || mapping.dim == DimBuffer)) + has_match = true; + if (dim != DimBuffer && mapping.dim != DimBuffer) + has_match = true; + } + else + { + has_match = true; + } + + if (has_match) + return join("set = ", mapping.desc_set, ", binding = ", mapping.binding); + } + } + + // Fallback to unknown mapping. + for (auto &mapping : descriptor_heap_mappings) + if (mapping.type == ResourceTypeUnknown) + return join("set = ", mapping.desc_set, ", binding = ", mapping.binding); + + return "descriptor_heap"; +} + diff --git a/src/libraries/spirv_cross/spirv_glsl.hpp b/src/libraries/spirv_cross/spirv_glsl.hpp index c04ef88bf..349fb8d3a 100644 --- a/src/libraries/spirv_cross/spirv_glsl.hpp +++ b/src/libraries/spirv_cross/spirv_glsl.hpp @@ -32,6 +32,9 @@ namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; +struct GlslConstantNameMapping; + enum PlsFormat { PlsNone = 0, @@ -66,7 +69,9 @@ enum AccessChainFlagBits ACCESS_CHAIN_SKIP_REGISTER_EXPRESSION_READ_BIT = 1 << 3, ACCESS_CHAIN_LITERAL_MSB_FORCE_ID = 1 << 4, ACCESS_CHAIN_FLATTEN_ALL_MEMBERS_BIT = 1 << 5, - ACCESS_CHAIN_FORCE_COMPOSITE_BIT = 1 << 6 + ACCESS_CHAIN_FORCE_COMPOSITE_BIT = 1 << 6, + ACCESS_CHAIN_PTR_CHAIN_POINTER_ARITH_BIT = 1 << 7, + ACCESS_CHAIN_PTR_CHAIN_CAST_TO_SCALAR_BIT = 1 << 8 }; typedef uint32_t AccessChainFlags; @@ -154,6 +159,9 @@ public: // If non-zero, controls layout(num_views = N) in; in GL_OVR_multiview2. uint32_t ovr_multiview_view_count = 0; + // Emit the entry point name in SPIR-V rather than "main". + bool use_entry_point_name = false; + enum Precision { DontCare, @@ -285,7 +293,7 @@ public: // This option is only meaningful for MSL and HLSL, since GLSL matches by location directly. // Masking builtins only takes effect if the builtin in question is part of the stage output interface. void mask_stage_output_by_location(uint32_t location, uint32_t component); - void mask_stage_output_by_builtin(spv::BuiltIn builtin); + void mask_stage_output_by_builtin(BuiltIn builtin); // Allow to control how to format float literals in the output. // Set to "nullptr" to use the default "convert_to_string" function. @@ -295,6 +303,22 @@ public: float_formatter = formatter; } + // Returns the macro name corresponding to constant id + std::string constant_value_macro_name(uint32_t id) const; + + // Rather than using layout(descriptor_heap), emit layout(set, binding). + // This intended to be compatible with descriptor buffers, legacy descriptor indexing, + // or when the heap descriptors require unusual kinds of mapping in the Vulkan API + // which is not expressible by GLSL directly. + // + // ResourceTypeUnknown can be used as a default catch-all mapping. + // dim can be used to disambiguate between texel buffers and images since they are both image types, + // but use different descriptor types in the Vulkan API. + // No distinction is made between 1D/2D/3D/Cube textures. + // The default argument of DimMax maps to both texel buffers and images. + // dim is ignored for ResourceTypeUnknown. + void remap_descriptor_heap(ResourceType type, uint32_t desc_set, uint32_t binding, Dim dim = DimMax); + protected: struct ShaderSubgroupSupportHelper { @@ -382,7 +406,7 @@ protected: }; // TODO remove this function when all subgroup ops are supported (or make it always return true) - static bool is_supported_subgroup_op_in_opengl(spv::Op op, const uint32_t *ops); + static bool is_supported_subgroup_op_in_opengl(Op op, const uint32_t *ops); void reset(uint32_t iteration_count); void emit_function(SPIRFunction &func, const Bitset &return_flags); @@ -409,7 +433,7 @@ protected: // For relax_nan_checks. GLSLstd450 get_remapped_glsl_op(GLSLstd450 std450_op) const; - spv::Op get_remapped_spirv_op(spv::Op op) const; + Op get_remapped_spirv_op(Op op) const; virtual void emit_glsl_op(uint32_t result_type, uint32_t result_id, uint32_t op, const uint32_t *args, uint32_t count); @@ -421,6 +445,8 @@ protected: const uint32_t *args, uint32_t count); virtual void emit_spv_amd_gcn_shader_op(uint32_t result_type, uint32_t result_id, uint32_t op, const uint32_t *args, uint32_t count); + void emit_non_semantic_shader_debug_info(uint32_t result_type, uint32_t result_id, uint32_t op, + const uint32_t *args, uint32_t count); virtual void emit_header(); void emit_line_directive(uint32_t file_id, uint32_t line_literal); void build_workgroup_size(SmallVector &arguments, const SpecializationConstant &x, @@ -434,11 +460,10 @@ protected: SmallVector &inherited_expressions); virtual void emit_subgroup_op(const Instruction &i); virtual std::string type_to_glsl(const SPIRType &type, uint32_t id = 0); - virtual std::string builtin_to_glsl(spv::BuiltIn builtin, spv::StorageClass storage); + virtual std::string builtin_to_glsl(BuiltIn builtin, StorageClass storage); virtual void emit_struct_member(const SPIRType &type, uint32_t member_type_id, uint32_t index, const std::string &qualifier = "", uint32_t base_offset = 0); - virtual void emit_struct_padding_target(const SPIRType &type); - virtual std::string image_type_glsl(const SPIRType &type, uint32_t id = 0); + virtual std::string image_type_glsl(const SPIRType &type, uint32_t id = 0, bool member = false); std::string constant_expression(const SPIRConstant &c, bool inside_block_like_struct_scope = false, bool inside_struct_scope = false); @@ -446,8 +471,9 @@ protected: virtual std::string constant_expression_vector(const SPIRConstant &c, uint32_t vector); virtual void emit_fixup(); virtual std::string variable_decl(const SPIRType &type, const std::string &name, uint32_t id = 0); - virtual bool variable_decl_is_remapped_storage(const SPIRVariable &var, spv::StorageClass storage) const; + virtual bool variable_decl_is_remapped_storage(const SPIRVariable &var, StorageClass storage) const; virtual std::string to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_t id); + virtual void emit_workgroup_initialization(const SPIRVariable &var); struct TextureFunctionBaseArguments { @@ -477,7 +503,7 @@ protected: uint32_t coord = 0, coord_components = 0, dref = 0; uint32_t grad_x = 0, grad_y = 0, lod = 0, offset = 0; uint32_t bias = 0, component = 0, sample = 0, sparse_texel = 0, min_lod = 0; - bool nonuniform_expression = false; + bool nonuniform_expression = false, has_array_offsets = false; }; virtual std::string to_function_args(const TextureFunctionArguments &args, bool *p_forward); @@ -490,7 +516,7 @@ protected: virtual std::string unpack_expression_type(std::string expr_str, const SPIRType &type, uint32_t physical_type_id, bool packed_type, bool row_major); - virtual bool builtin_translates_to_nonarray(spv::BuiltIn builtin) const; + virtual bool builtin_translates_to_nonarray(BuiltIn builtin) const; virtual bool is_user_type_structured(uint32_t id) const; @@ -564,8 +590,8 @@ protected: Options options; - virtual std::string type_to_array_glsl( - const SPIRType &type); // Allow Metal to use the array template to make arrays a value type + // Allow Metal to use the array template to make arrays a value type + virtual std::string type_to_array_glsl(const SPIRType &type, uint32_t variable_id); std::string to_array_size(const SPIRType &type, uint32_t index); uint32_t to_array_size_literal(const SPIRType &type, uint32_t index) const; uint32_t to_array_size_literal(const SPIRType &type) const; @@ -620,6 +646,8 @@ protected: const char *uint16_t_literal_suffix = "us"; const char *nonuniform_qualifier = "nonuniformEXT"; const char *boolean_mix_function = "mix"; + const char *printf_function = "debugPrintfEXT"; + std::string constant_null_initializer = ""; SPIRType::BaseType boolean_in_struct_remapped_type = SPIRType::Boolean; bool swizzle_is_function = false; bool shared_is_implied = false; @@ -627,6 +655,7 @@ protected: bool explicit_struct_type = false; bool use_initializer_list = false; bool use_typed_initializer_list = false; + bool requires_matching_array_initializer = false; bool can_declare_struct_inline = true; bool can_declare_arrays_inline = true; bool native_row_major_matrix = true; @@ -653,20 +682,24 @@ protected: bool workgroup_size_is_hidden = false; bool requires_relaxed_precision_analysis = false; bool implicit_c_integer_promotion_rules = false; + bool supports_spec_constant_array_size = true; + bool requires_phi_undef_zero_init = false; } backend; void emit_struct(SPIRType &type); void emit_resources(); - void emit_extension_workarounds(spv::ExecutionModel model); - void emit_subgroup_arithmetic_workaround(const std::string &func, spv::Op op, spv::GroupOperation group_op); + void emit_extension_workarounds(ExecutionModel model); + void emit_subgroup_arithmetic_workaround(const std::string &func, Op op, GroupOperation group_op); void emit_polyfills(uint32_t polyfills, bool relaxed); - void emit_buffer_block_native(const SPIRVariable &var); + void emit_buffer_block_native(const SPIRVariable *var, const DescriptorHeapMeta *heap_meta = nullptr); + std::string to_buffer_pointer_name_prefix(uint32_t ptr_id) const; + static std::string heap_meta_to_prefix(const DescriptorHeapMeta &meta); void emit_buffer_reference_block(uint32_t type_id, bool forward_declaration); void emit_buffer_block_legacy(const SPIRVariable &var); void emit_buffer_block_flattened(const SPIRVariable &type); - void fixup_implicit_builtin_block_names(spv::ExecutionModel model); - void emit_declared_builtin_block(spv::StorageClass storage, spv::ExecutionModel model); - bool should_force_emit_builtin_block(spv::StorageClass storage); + void fixup_implicit_builtin_block_names(ExecutionModel model); + void emit_declared_builtin_block(StorageClass storage, ExecutionModel model); + bool should_force_emit_builtin_block(StorageClass storage); void emit_push_constant_block_vulkan(const SPIRVariable &var); void emit_push_constant_block_glsl(const SPIRVariable &var); void emit_interface_block(const SPIRVariable &type); @@ -676,8 +709,9 @@ protected: void emit_flattened_io_block_member(const std::string &basename, const SPIRType &type, const char *qual, const SmallVector &indices); void emit_block_chain(SPIRBlock &block); + BlockID emit_block_chain_inner(SPIRBlock &block); + void emit_block_chain_cleanup(SPIRBlock &block); void emit_hoisted_temporaries(SmallVector> &temporaries); - std::string constant_value_macro_name(uint32_t id); int get_constant_mapping_to_workgroup_component(const SPIRConstant &constant) const; void emit_constant(const SPIRConstant &constant); void emit_specialization_constant_op(const SPIRConstantOp &constant); @@ -693,6 +727,7 @@ protected: void emit_variable_temporary_copies(const SPIRVariable &var); bool should_dereference(uint32_t id); + bool should_dereference_caller_param(uint32_t id); bool should_forward(uint32_t id) const; bool should_suppress_usage_tracking(uint32_t id) const; void emit_mix_op(uint32_t result_type, uint32_t id, uint32_t left, uint32_t right, uint32_t lerp); @@ -746,22 +781,28 @@ protected: bool expression_read_implies_multiple_reads(uint32_t id) const; SPIRExpression &emit_op(uint32_t result_type, uint32_t result_id, const std::string &rhs, bool forward_rhs, bool suppress_usage_tracking = false); + void emit_transposed_op(uint32_t result_type, uint32_t result_id, const std::string &rhs, bool forward_rhs); void access_chain_internal_append_index(std::string &expr, uint32_t base, const SPIRType *type, AccessChainFlags flags, bool &access_chain_is_arrayed, uint32_t index); std::string access_chain_internal(uint32_t base, const uint32_t *indices, uint32_t count, AccessChainFlags flags, - AccessChainMeta *meta); + AccessChainMeta *meta, const SPIRType *untyped_data_type); - spv::StorageClass get_expression_effective_storage_class(uint32_t ptr); + // Only meaningful on backends with physical pointer support ala MSL. + // Relevant for PtrAccessChain / BDA. + virtual uint32_t get_physical_type_id_stride(TypeID type_id) const; + + StorageClass get_expression_effective_storage_class(uint32_t ptr); virtual bool access_chain_needs_stage_io_builtin_translation(uint32_t base); - virtual void check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type); + virtual bool check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type); virtual bool prepare_access_chain_for_scalar_access(std::string &expr, const SPIRType &type, - spv::StorageClass storage, bool &is_packed); + StorageClass storage, bool &is_packed); std::string access_chain(uint32_t base, const uint32_t *indices, uint32_t count, const SPIRType &target_type, - AccessChainMeta *meta = nullptr, bool ptr_chain = false); + AccessChainMeta *meta = nullptr, bool ptr_chain = false, + const SPIRType *untyped_data_type = nullptr); std::string flattened_access_chain(uint32_t base, const uint32_t *indices, uint32_t count, const SPIRType &target_type, uint32_t offset, uint32_t matrix_stride, @@ -784,10 +825,16 @@ protected: const char *index_to_swizzle(uint32_t index); std::string remap_swizzle(const SPIRType &result_type, uint32_t input_components, const std::string &expr); std::string declare_temporary(uint32_t type, uint32_t id); + bool can_declare_inline_temporary(uint32_t id) const; void emit_uninitialized_temporary(uint32_t type, uint32_t id); SPIRExpression &emit_uninitialized_temporary_expression(uint32_t type, uint32_t id); - void append_global_func_args(const SPIRFunction &func, uint32_t index, SmallVector &arglist); + virtual void append_global_func_args(const SPIRFunction &func, uint32_t index, SmallVector &arglist); std::string to_non_uniform_aware_expression(uint32_t id); + std::string to_atomic_ptr_expression(uint32_t id); + std::string to_pretty_expression_if_int_constant( + uint32_t id, + const GlslConstantNameMapping *mapping_start, const GlslConstantNameMapping *mapping_end, + bool register_expression_read = true); std::string to_expression(uint32_t id, bool register_expression_read = true); std::string to_composite_constructor_expression(const SPIRType &parent_type, uint32_t id, bool block_like_type); std::string to_rerolled_array_expression(const SPIRType &parent_type, const std::string &expr, const SPIRType &type); @@ -816,15 +863,15 @@ protected: void emit_output_variable_initializer(const SPIRVariable &var); std::string to_precision_qualifiers_glsl(uint32_t id); virtual const char *to_storage_qualifiers_glsl(const SPIRVariable &var); - std::string flags_to_qualifiers_glsl(const SPIRType &type, const Bitset &flags); - const char *format_to_glsl(spv::ImageFormat format); + std::string flags_to_qualifiers_glsl(const SPIRType &type, uint32_t id, const Bitset &flags); + const char *format_to_glsl(ImageFormat format); virtual std::string layout_for_member(const SPIRType &type, uint32_t index); virtual std::string to_interpolation_qualifiers(const Bitset &flags); std::string layout_for_variable(const SPIRVariable &variable); std::string to_combined_image_sampler(VariableID image_id, VariableID samp_id); virtual bool skip_argument(uint32_t id) const; virtual bool emit_array_copy(const char *expr, uint32_t lhs_id, uint32_t rhs_id, - spv::StorageClass lhs_storage, spv::StorageClass rhs_storage); + StorageClass lhs_storage, StorageClass rhs_storage); virtual void emit_block_hints(const SPIRBlock &block); virtual std::string to_initializer_expression(const SPIRVariable &var); virtual std::string to_zero_initialized_expression(uint32_t type_id); @@ -833,7 +880,9 @@ protected: bool buffer_is_packing_standard(const SPIRType &type, BufferPackingStandard packing, uint32_t *failed_index = nullptr, uint32_t start_offset = 0, uint32_t end_offset = ~(0u)); - std::string buffer_to_packing_standard(const SPIRType &type, bool support_std430_without_scalar_layout); + std::string buffer_to_packing_standard(const SPIRType &type, + bool support_std430_without_scalar_layout, + bool support_enhanced_layouts); uint32_t type_to_packed_base_size(const SPIRType &type, BufferPackingStandard packing); uint32_t type_to_packed_alignment(const SPIRType &type, const Bitset &flags, BufferPackingStandard packing); @@ -931,12 +980,32 @@ protected: PolyfillMatrixInverse2x2 = 1 << 6, PolyfillMatrixInverse3x3 = 1 << 7, PolyfillMatrixInverse4x4 = 1 << 8, + PolyfillNMin16 = 1 << 9, + PolyfillNMin32 = 1 << 10, + PolyfillNMin64 = 1 << 11, + PolyfillNMax16 = 1 << 12, + PolyfillNMax32 = 1 << 13, + PolyfillNMax64 = 1 << 14, + PolyfillNClamp16 = 1 << 15, + PolyfillNClamp32 = 1 << 16, + PolyfillNClamp64 = 1 << 17, }; uint32_t required_polyfills = 0; uint32_t required_polyfills_relaxed = 0; void require_polyfill(Polyfill polyfill, bool relaxed); + struct IntegerDotProduct + { + Id result_type; + Id argument_type[2]; + Op op; + }; + SmallVector integer_dot_products_polyfills; + void add_integer_dot_product_polyfill(const IntegerDotProduct &idot); + std::string integer_dot_product_entry_point(const IntegerDotProduct &idot); + void emit_polyfills_integer_dot_product(); + bool ray_tracing_is_khr = false; bool barycentric_is_nv = false; void ray_tracing_khr_fixup_locations(); @@ -982,16 +1051,19 @@ protected: std::string emit_for_loop_initializers(const SPIRBlock &block); void emit_while_loop_initializers(const SPIRBlock &block); + std::string undef_loop_variable_initializer_suffix(const SPIRVariable &var); bool for_loop_initializers_are_same_type(const SPIRBlock &block); bool optimize_read_modify_write(const SPIRType &type, const std::string &lhs, const std::string &rhs); void fixup_image_load_store_access(); bool type_is_empty(const SPIRType &type); - bool can_use_io_location(spv::StorageClass storage, bool block); + bool can_use_io_location(StorageClass storage, bool block); const Instruction *get_next_instruction_in_block(const Instruction &instr); static uint32_t mask_relevant_memory_semantics(uint32_t semantics); + std::string convert_floate4m3_to_string(const SPIRConstant &value, uint32_t col, uint32_t row); + std::string convert_floate5m2_to_string(const SPIRConstant &value, uint32_t col, uint32_t row); std::string convert_half_to_string(const SPIRConstant &value, uint32_t col, uint32_t row); std::string convert_float_to_string(const SPIRConstant &value, uint32_t col, uint32_t row); std::string convert_double_to_string(const SPIRConstant &value, uint32_t col, uint32_t row); @@ -1001,7 +1073,7 @@ protected: // Builtins in GLSL are always specific signedness, but the SPIR-V can declare them // as either unsigned or signed. // Sometimes we will need to automatically perform casts on load and store to make this work. - virtual SPIRType::BaseType get_builtin_basetype(spv::BuiltIn builtin, SPIRType::BaseType default_type); + virtual SPIRType::BaseType get_builtin_basetype(BuiltIn builtin, SPIRType::BaseType default_type); virtual void cast_to_variable_store(uint32_t target_id, std::string &expr, const SPIRType &expr_type); virtual void cast_from_variable_load(uint32_t source_id, std::string &expr, const SPIRType &expr_type); void unroll_array_from_complex_load(uint32_t target_id, uint32_t source_id, std::string &expr); @@ -1012,7 +1084,7 @@ protected: void disallow_forwarding_in_expression_chain(const SPIRExpression &expr); bool expression_is_constant_null(uint32_t id) const; - bool expression_is_non_value_type_array(uint32_t ptr); + bool expression_is_non_value_type_array(uint32_t value_type_id, uint32_t ptr); virtual void emit_store_statement(uint32_t lhs_expression, uint32_t rhs_expression); uint32_t get_integer_width_for_instruction(const Instruction &instr) const; @@ -1030,7 +1102,7 @@ protected: static const char *vector_swizzle(int vecsize, int index); bool is_stage_output_location_masked(uint32_t location, uint32_t component) const; - bool is_stage_output_builtin_masked(spv::BuiltIn builtin) const; + bool is_stage_output_builtin_masked(BuiltIn builtin) const; bool is_stage_output_variable_masked(const SPIRVariable &var) const; bool is_stage_output_block_member_masked(const SPIRVariable &var, uint32_t index, bool strip_array) const; bool is_per_primitive_variable(const SPIRVariable &var) const; @@ -1043,6 +1115,20 @@ protected: std::string format_float(float value) const; std::string format_double(double value) const; + uint32_t get_fp_fast_math_flags_for_op(uint32_t result_type, uint32_t id) const; + bool has_legacy_nocontract(uint32_t result_type, uint32_t id) const; + + struct DescriptorHeapMapping + { + ResourceType type; + uint32_t desc_set; + uint32_t binding; + Dim dim; + }; + SmallVector descriptor_heap_mappings; + bool is_descriptor_non_uniform(uint32_t id) const; + std::string to_descriptor_heap_layout(const SPIRType &type, StorageClass storage = StorageClassUniformConstant) const; + private: void init(); diff --git a/src/libraries/spirv_cross/spirv_hlsl.cpp b/src/libraries/spirv_cross/spirv_hlsl.cpp index 9b8349365..635527978 100644 --- a/src/libraries/spirv_cross/spirv_hlsl.cpp +++ b/src/libraries/spirv_cross/spirv_hlsl.cpp @@ -27,7 +27,7 @@ #include #include -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; using namespace std; @@ -491,9 +491,9 @@ string CompilerHLSL::type_to_glsl(const SPIRType &type, uint32_t id) case SPIRType::Double: return join("double", type.vecsize); case SPIRType::Int64: - return join("i64vec", type.vecsize); + return join("int64_t", type.vecsize); case SPIRType::UInt64: - return join("u64vec", type.vecsize); + return join("uint64_t", type.vecsize); default: return "???"; } @@ -769,6 +769,10 @@ void CompilerHLSL::emit_builtin_inputs_in_struct() auto builtin = static_cast(i); switch (builtin) { + case BuiltInPosition: + type = "float4"; + semantic = legacy ? "POSITION" : "SV_Position"; + break; case BuiltInFragCoord: type = "float4"; semantic = legacy ? "VPOS" : "SV_Position"; @@ -783,8 +787,27 @@ void CompilerHLSL::emit_builtin_inputs_in_struct() break; case BuiltInPrimitiveId: - type = "uint"; - semantic = "SV_PrimitiveID"; + // For geometry shaders, PrimitiveId is a direct function parameter + // (SV_PrimitiveID), not part of the input struct. + if (get_entry_point().model != ExecutionModelGeometry) + { + type = "uint"; + semantic = "SV_PrimitiveID"; + } + break; + + case BuiltInInvocationId: + if (get_entry_point().model == ExecutionModelGeometry) + { + type = "uint"; + semantic = "SV_GSInstanceID"; + } + else if (get_entry_point().model != ExecutionModelTessellationControl) + { + // For tesc, InvocationId is a direct function parameter (SV_OutputControlPointID), + // not part of the input struct. + SPIRV_CROSS_THROW("InvocationId is only supported in geometry and tessellation control shaders."); + } break; case BuiltInInstanceId: @@ -849,11 +872,25 @@ void CompilerHLSL::emit_builtin_inputs_in_struct() case BuiltInSubgroupLeMask: case BuiltInSubgroupGtMask: case BuiltInSubgroupGeMask: - case BuiltInBaseVertex: - case BuiltInBaseInstance: // Handled specially. break; + case BuiltInBaseVertex: + if (hlsl_options.shader_model >= 68) + { + type = "uint"; + semantic = "SV_StartVertexLocation"; + } + break; + + case BuiltInBaseInstance: + if (hlsl_options.shader_model >= 68) + { + type = "uint"; + semantic = "SV_StartInstanceLocation"; + } + break; + case BuiltInHelperInvocation: if (hlsl_options.shader_model < 50 || get_entry_point().model != ExecutionModelFragment) SPIRV_CROSS_THROW("Helper Invocation input is only supported in PS 5.0 or higher."); @@ -905,6 +942,17 @@ void CompilerHLSL::emit_builtin_inputs_in_struct() semantic = "SV_RenderTargetArrayIndex"; break; + case BuiltInBaryCoordKHR: + case BuiltInBaryCoordNoPerspKHR: + if (hlsl_options.shader_model < 61) + SPIRV_CROSS_THROW("SM 6.1 is required for barycentrics."); + type = builtin == BuiltInBaryCoordNoPerspKHR ? "noperspective float3" : "float3"; + if (active_input_builtins.get(BuiltInBaryCoordKHR) && active_input_builtins.get(BuiltInBaryCoordNoPerspKHR)) + semantic = builtin == BuiltInBaryCoordKHR ? "SV_Barycentrics0" : "SV_Barycentrics1"; + else + semantic = "SV_Barycentrics"; + break; + default: SPIRV_CROSS_THROW("Unsupported builtin in HLSL."); } @@ -944,7 +992,7 @@ string CompilerHLSL::to_interpolation_qualifiers(const Bitset &flags) string res; //if (flags & (1ull << DecorationSmooth)) // res += "linear "; - if (flags.get(DecorationFlat)) + if (flags.get(DecorationFlat) || flags.get(DecorationPerVertexKHR)) res += "nointerpolation "; if (flags.get(DecorationNoPerspective)) res += "noperspective "; @@ -996,13 +1044,23 @@ void CompilerHLSL::emit_interface_block_member_in_struct(const SPIRVariable &var { auto &execution = get_entry_point(); auto type = get(var.basetype); - auto semantic = to_semantic(location, execution.model, var.storage); + + std::string semantic; + if (hlsl_options.user_semantic && has_member_decoration(var.self, member_index, DecorationUserSemantic)) + semantic = get_member_decoration_string(var.self, member_index, DecorationUserSemantic); + else + semantic = to_semantic(location, execution.model, var.storage); + auto mbr_name = join(to_name(type.self), "_", to_member_name(type, member_index)); auto &mbr_type = get(type.member_types[member_index]); - statement(to_interpolation_qualifiers(get_member_decoration_bitset(type.self, member_index)), + Bitset member_decorations = get_member_decoration_bitset(type.self, member_index); + if (has_decoration(var.self, DecorationPerVertexKHR)) + member_decorations.set(DecorationPerVertexKHR); + + statement(to_interpolation_qualifiers(member_decorations), type_to_glsl(mbr_type), - " ", mbr_name, type_to_array_glsl(mbr_type), + " ", mbr_name, type_to_array_glsl(mbr_type, var.self), " : ", semantic, ";"); // Structs and arrays should consume more locations. @@ -1051,17 +1109,28 @@ void CompilerHLSL::emit_interface_block_in_struct(const SPIRVariable &var, unord auto name = to_name(var.self); if (use_location_number) { - uint32_t location_number; + uint32_t location_number = UINT32_MAX; - // If an explicit location exists, use it with TEXCOORD[N] semantic. - // Otherwise, pick a vacant location. - if (has_decoration(var.self, DecorationLocation)) - location_number = get_decoration(var.self, DecorationLocation); + std::string semantic; + bool has_user_semantic = false; + + if (hlsl_options.user_semantic && has_decoration(var.self, DecorationUserSemantic)) + { + semantic = get_decoration_string(var.self, DecorationUserSemantic); + has_user_semantic = true; + } else - location_number = get_vacant_location(); + { + // If an explicit location exists, use it with TEXCOORD[N] semantic. + // Otherwise, pick a vacant location. + if (has_decoration(var.self, DecorationLocation)) + location_number = get_decoration(var.self, DecorationLocation); + else + location_number = get_vacant_location(); - // Allow semantic remap if specified. - auto semantic = to_semantic(location_number, execution.model, var.storage); + // Allow semantic remap if specified. + semantic = to_semantic(location_number, execution.model, var.storage); + } if (need_matrix_unroll && type.columns > 1) { @@ -1075,31 +1144,38 @@ void CompilerHLSL::emit_interface_block_in_struct(const SPIRVariable &var, unord newtype.columns = 1; string effective_semantic; - if (hlsl_options.flatten_matrix_vertex_input_semantics) + if (hlsl_options.flatten_matrix_vertex_input_semantics && !has_user_semantic) effective_semantic = to_semantic(location_number, execution.model, var.storage); else effective_semantic = join(semantic, "_", i); statement(to_interpolation_qualifiers(get_decoration_bitset(var.self)), variable_decl(newtype, join(name, "_", i)), " : ", effective_semantic, ";"); - active_locations.insert(location_number++); + if (location_number != UINT32_MAX) + active_locations.insert(location_number++); } } else { auto decl_type = type; - if (execution.model == ExecutionModelMeshEXT) + if (execution.model == ExecutionModelMeshEXT || + (execution.model == ExecutionModelGeometry && var.storage == StorageClassInput) || + has_decoration(var.self, DecorationPerVertexKHR)) { - decl_type.array.erase(decl_type.array.begin()); - decl_type.array_size_literal.erase(decl_type.array_size_literal.begin()); + // The per-vertex/per-CP dimension is the outermost (last element in array vector). + decl_type.array.pop_back(); + decl_type.array_size_literal.pop_back(); } statement(to_interpolation_qualifiers(get_decoration_bitset(var.self)), variable_decl(decl_type, name), " : ", semantic, ";"); - // Structs and arrays should consume more locations. - uint32_t consumed_locations = type_to_consumed_locations(decl_type); - for (uint32_t i = 0; i < consumed_locations; i++) - active_locations.insert(location_number + i); + if (location_number != UINT32_MAX) + { + // Structs and arrays should consume more locations. + uint32_t consumed_locations = type_to_consumed_locations(decl_type); + for (uint32_t i = 0; i < consumed_locations; i++) + active_locations.insert(location_number + i); + } } } else @@ -1108,10 +1184,13 @@ void CompilerHLSL::emit_interface_block_in_struct(const SPIRVariable &var, unord } } -std::string CompilerHLSL::builtin_to_glsl(spv::BuiltIn builtin, spv::StorageClass storage) +std::string CompilerHLSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) { switch (builtin) { + case BuiltInPosition: + // We want to avoid clash between input/output for geometry shader + return storage == StorageClass::StorageClassInput ? "gl_PositionIn" : "gl_Position"; case BuiltInVertexId: return "gl_VertexID"; case BuiltInInstanceId: @@ -1195,9 +1274,7 @@ void CompilerHLSL::emit_builtin_variables() // Emit global variables for the interface variables which are statically used by the shader. builtins.for_each_bit([&](uint32_t i) { - const char *type = nullptr; auto builtin = static_cast(i); - uint32_t array_size = 0; string init_expr; auto init_itr = builtin_to_initializer.find(builtin); @@ -1216,144 +1293,167 @@ void CompilerHLSL::emit_builtin_variables() } } - switch (builtin) + // If we need to emit 2 separate variables (for both input & output), we'll update this value + bool has_separate_input_output = false; + for (int variable_index = 0; variable_index < (has_separate_input_output ? 2 : 1); variable_index++) { - case BuiltInFragCoord: - case BuiltInPosition: - type = "float4"; - break; - - case BuiltInFragDepth: - type = "float"; - break; - - case BuiltInVertexId: - case BuiltInVertexIndex: - case BuiltInInstanceIndex: - type = "int"; - if (hlsl_options.support_nonzero_base_vertex_base_instance) - base_vertex_info.used = true; - break; - - case BuiltInBaseVertex: - case BuiltInBaseInstance: - type = "int"; - base_vertex_info.used = true; - break; - - case BuiltInInstanceId: - case BuiltInSampleId: - type = "int"; - break; - - case BuiltInPointSize: - if (hlsl_options.point_size_compat || hlsl_options.shader_model <= 30) + uint32_t array_size = 0; + StorageClass storage = active_input_builtins.get(i) && variable_index == 0 + ? StorageClassInput + : StorageClassOutput; + const char *type = nullptr; + switch (builtin) { - // Just emit the global variable, it will be ignored. + case BuiltInFragCoord: + type = "float4"; + break; + + case BuiltInPosition: + type = "float4"; + if (storage == StorageClass::StorageClassInput && + (get_execution_model() == ExecutionModelGeometry || + get_execution_model() == ExecutionModelTessellationControl)) + array_size = input_vertices_from_execution_mode(get_entry_point()); + break; + + case BuiltInFragDepth: type = "float"; break; - } - else + + case BuiltInVertexId: + case BuiltInVertexIndex: + case BuiltInInstanceIndex: + type = "int"; + if (hlsl_options.support_nonzero_base_vertex_base_instance || hlsl_options.shader_model >= 68) + base_vertex_info.used = true; + break; + + case BuiltInBaseVertex: + case BuiltInBaseInstance: + type = "int"; + base_vertex_info.used = true; + break; + + case BuiltInInstanceId: + case BuiltInSampleId: + type = "int"; + break; + + case BuiltInPointSize: + if (hlsl_options.point_size_compat || hlsl_options.shader_model <= 30) + { + // Just emit the global variable, it will be ignored. + type = "float"; + break; + } + else + SPIRV_CROSS_THROW(join("Unsupported builtin in HLSL: ", unsigned(builtin))); + + case BuiltInGlobalInvocationId: + case BuiltInLocalInvocationId: + case BuiltInWorkgroupId: + type = "uint3"; + break; + + case BuiltInLocalInvocationIndex: + type = "uint"; + break; + + case BuiltInFrontFacing: + type = "bool"; + break; + + case BuiltInNumWorkgroups: + case BuiltInPointCoord: + // Handled specially. + break; + + case BuiltInSubgroupLocalInvocationId: + case BuiltInSubgroupSize: + if (hlsl_options.shader_model < 60) + SPIRV_CROSS_THROW("Need SM 6.0 for Wave ops."); + break; + + case BuiltInSubgroupEqMask: + case BuiltInSubgroupLtMask: + case BuiltInSubgroupLeMask: + case BuiltInSubgroupGtMask: + case BuiltInSubgroupGeMask: + if (hlsl_options.shader_model < 60) + SPIRV_CROSS_THROW("Need SM 6.0 for Wave ops."); + type = "uint4"; + break; + + case BuiltInHelperInvocation: + if (hlsl_options.shader_model < 50) + SPIRV_CROSS_THROW("Need SM 5.0 for Helper Invocation."); + break; + + case BuiltInClipDistance: + array_size = clip_distance_count; + type = "float"; + break; + + case BuiltInCullDistance: + array_size = cull_distance_count; + type = "float"; + break; + + case BuiltInSampleMask: + if (storage == StorageClass::StorageClassInput) + type = sample_mask_in_basetype == SPIRType::UInt ? "uint" : "int"; + else + type = sample_mask_out_basetype == SPIRType::UInt ? "uint" : "int"; + array_size = 1; + break; + + case BuiltInPrimitiveId: + case BuiltInViewIndex: + case BuiltInLayer: + type = "uint"; + break; + + case BuiltInViewportIndex: + case BuiltInPrimitiveShadingRateKHR: + case BuiltInPrimitiveLineIndicesEXT: + case BuiltInCullPrimitiveEXT: + type = "uint"; + break; + + case BuiltInBaryCoordKHR: + case BuiltInBaryCoordNoPerspKHR: + if (hlsl_options.shader_model < 61) + SPIRV_CROSS_THROW("Need SM 6.1 for barycentrics."); + type = "float3"; + break; + + default: SPIRV_CROSS_THROW(join("Unsupported builtin in HLSL: ", unsigned(builtin))); + } - case BuiltInGlobalInvocationId: - case BuiltInLocalInvocationId: - case BuiltInWorkgroupId: - type = "uint3"; - break; + if (type) + { + auto builtin_name = builtin_to_glsl(builtin, storage); + if (array_size) + statement("static ", type, " ", builtin_name, "[", array_size, "]", init_expr, ";"); + else + statement("static ", type, " ", builtin_name, init_expr, ";"); - case BuiltInLocalInvocationIndex: - type = "uint"; - break; - - case BuiltInFrontFacing: - type = "bool"; - break; - - case BuiltInNumWorkgroups: - case BuiltInPointCoord: - // Handled specially. - break; - - case BuiltInSubgroupLocalInvocationId: - case BuiltInSubgroupSize: - if (hlsl_options.shader_model < 60) - SPIRV_CROSS_THROW("Need SM 6.0 for Wave ops."); - break; - - case BuiltInSubgroupEqMask: - case BuiltInSubgroupLtMask: - case BuiltInSubgroupLeMask: - case BuiltInSubgroupGtMask: - case BuiltInSubgroupGeMask: - if (hlsl_options.shader_model < 60) - SPIRV_CROSS_THROW("Need SM 6.0 for Wave ops."); - type = "uint4"; - break; - - case BuiltInHelperInvocation: - if (hlsl_options.shader_model < 50) - SPIRV_CROSS_THROW("Need SM 5.0 for Helper Invocation."); - break; - - case BuiltInClipDistance: - array_size = clip_distance_count; - type = "float"; - break; - - case BuiltInCullDistance: - array_size = cull_distance_count; - type = "float"; - break; - - case BuiltInSampleMask: - if (active_input_builtins.get(BuiltInSampleMask)) - type = sample_mask_in_basetype == SPIRType::UInt ? "uint" : "int"; - else - type = sample_mask_out_basetype == SPIRType::UInt ? "uint" : "int"; - array_size = 1; - break; - - case BuiltInPrimitiveId: - case BuiltInViewIndex: - case BuiltInLayer: - type = "uint"; - break; - - case BuiltInViewportIndex: - case BuiltInPrimitiveShadingRateKHR: - case BuiltInPrimitiveLineIndicesEXT: - case BuiltInCullPrimitiveEXT: - type = "uint"; - break; - - default: - SPIRV_CROSS_THROW(join("Unsupported builtin in HLSL: ", unsigned(builtin))); - } - - StorageClass storage = active_input_builtins.get(i) ? StorageClassInput : StorageClassOutput; - - if (type) - { - if (array_size) - statement("static ", type, " ", builtin_to_glsl(builtin, storage), "[", array_size, "]", init_expr, ";"); - else - statement("static ", type, " ", builtin_to_glsl(builtin, storage), init_expr, ";"); - } - - // SampleMask can be both in and out with sample builtin, in this case we have already - // declared the input variable and we need to add the output one now. - if (builtin == BuiltInSampleMask && storage == StorageClassInput && this->active_output_builtins.get(i)) - { - type = sample_mask_out_basetype == SPIRType::UInt ? "uint" : "int"; - if (array_size) - statement("static ", type, " ", this->builtin_to_glsl(builtin, StorageClassOutput), "[", array_size, "]", init_expr, ";"); - else - statement("static ", type, " ", this->builtin_to_glsl(builtin, StorageClassOutput), init_expr, ";"); + if (storage == StorageClassInput && this->active_output_builtins.get(i)) + { + auto out_builtin_name = builtin_to_glsl(builtin, StorageClassOutput); + if (out_builtin_name != builtin_name) + { + // If built-in name differs, we need to output it again + // (we reevaluate type and array size in case they are different) + has_separate_input_output = true; + } + } + } } }); - if (base_vertex_info.used) + if (base_vertex_info.used && hlsl_options.shader_model < 68) { string binding_info; if (base_vertex_info.explicit_binding) @@ -1557,6 +1657,7 @@ void CompilerHLSL::replace_illegal_names() "Texture3D", "TextureCube", "TextureCubeArray", "true", "typedef", "triangle", "triangleadj", "TriangleStream", "uint", "uniform", "unorm", "unsigned", "vector", "vertexfragment", "VertexShader", "vertices", "void", "volatile", "while", + "signed", }; CompilerGLSL::replace_illegal_names(keywords); @@ -1671,9 +1772,11 @@ void CompilerHLSL::emit_resources() ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { auto &type = this->get(var.basetype); + bool is_hidden = is_hidden_io_variable(var); + if (var.storage != StorageClassFunction && !var.remapped_variable && type.pointer && (var.storage == StorageClassInput || var.storage == StorageClassOutput) && !is_builtin_variable(var) && - interface_variable_exists_in_entry_point(var.self)) + interface_variable_exists_in_entry_point(var.self) && !is_hidden) { // Builtin variables are handled separately. emit_interface_block_globally(var); @@ -1709,8 +1812,10 @@ void CompilerHLSL::emit_resources() if (var.storage != StorageClassInput && var.storage != StorageClassOutput) return; + bool is_hidden = is_hidden_io_variable(var); + if (!var.remapped_variable && type.pointer && !is_builtin_variable(var) && - interface_variable_exists_in_entry_point(var.self)) + interface_variable_exists_in_entry_point(var.self) && !is_hidden) { if (block) { @@ -1798,7 +1903,7 @@ void CompilerHLSL::emit_resources() if (!output_variables.empty() || !active_output_builtins.empty()) { sort(output_variables.begin(), output_variables.end(), variable_compare); - require_output = !is_mesh_shader; + require_output = !(is_mesh_shader || execution.model == ExecutionModelGeometry); statement(is_mesh_shader ? "struct gl_MeshPerVertexEXT" : "struct SPIRV_Cross_Output"); begin_scope(); @@ -2277,7 +2382,7 @@ void CompilerHLSL::emit_resources() // Need out variable since HLSL does not support returning arrays. auto &type = get(type_id); auto type_str = type_to_glsl(type); - auto type_arr_str = type_to_array_glsl(type); + auto type_arr_str = type_to_array_glsl(type, 0); statement("void spvSelectComposite(out ", type_str, " out_value", type_arr_str, ", bool cond, ", type_str, " true_val", type_arr_str, ", ", type_str, " false_val", type_arr_str, ")"); @@ -2440,12 +2545,14 @@ void CompilerHLSL::analyze_meshlet_writes() set_decoration(op_type, DecorationPerPrimitiveEXT); auto &arr = set(op_arr, type); + arr.op = OpTypeArray; arr.parent_type = type.self; arr.array.push_back(per_primitive ? execution.output_primitives : execution.output_vertices); arr.array_size_literal.push_back(true); auto &ptr = set(op_ptr, arr); ptr.parent_type = arr.self; + ptr.op = OpTypePointer; ptr.pointer = true; ptr.pointer_depth++; ptr.storage = StorageClassOutput; @@ -2640,6 +2747,83 @@ void CompilerHLSL::emit_mesh_tasks(SPIRBlock &block) } } +void CompilerHLSL::emit_geometry_stream_append() +{ + begin_scope(); + statement("SPIRV_Cross_Output stage_output;"); + + active_output_builtins.for_each_bit( + [&](uint32_t i) + { + if (i == BuiltInPointSize && hlsl_options.shader_model > 30) + return; + switch (static_cast(i)) + { + case BuiltInClipDistance: + for (uint32_t clip = 0; clip < clip_distance_count; clip++) + statement("stage_output.gl_ClipDistance", clip / 4, ".", "xyzw"[clip & 3], " = gl_ClipDistance[", + clip, "];"); + break; + case BuiltInCullDistance: + for (uint32_t cull = 0; cull < cull_distance_count; cull++) + statement("stage_output.gl_CullDistance", cull / 4, ".", "xyzw"[cull & 3], " = gl_CullDistance[", + cull, "];"); + break; + case BuiltInSampleMask: + statement("stage_output.gl_SampleMask = gl_SampleMask[0];"); + break; + default: + { + auto builtin_expr = builtin_to_glsl(static_cast(i), StorageClassOutput); + statement("stage_output.", builtin_expr, " = ", builtin_expr, ";"); + } + break; + } + }); + + ir.for_each_typed_id( + [&](uint32_t, SPIRVariable &var) + { + auto &type = this->get(var.basetype); + bool block = has_decoration(type.self, DecorationBlock); + + if (var.storage != StorageClassOutput) + return; + + if (!var.remapped_variable && type.pointer && !is_builtin_variable(var) && + interface_variable_exists_in_entry_point(var.self)) + { + if (block) + { + auto type_name = to_name(type.self); + auto var_name = to_name(var.self); + for (uint32_t mbr_idx = 0; mbr_idx < uint32_t(type.member_types.size()); mbr_idx++) + { + auto mbr_name = to_member_name(type, mbr_idx); + auto flat_name = join(type_name, "_", mbr_name); + statement("stage_output.", flat_name, " = ", var_name, ".", mbr_name, ";"); + } + } + else + { + auto name = to_name(var.self); + if (hlsl_options.shader_model <= 30 && get_entry_point().model == ExecutionModelFragment) + { + string output_filler; + for (uint32_t size = type.vecsize; size < 4; ++size) + output_filler += ", 0.0"; + statement("stage_output.", name, " = float4(", name, output_filler, ");"); + } + else + statement("stage_output.", name, " = ", name, ";"); + } + } + }); + + statement("geometry_stream.Append(stage_output);"); + end_scope(); +} + void CompilerHLSL::emit_buffer_block(const SPIRVariable &var) { auto &type = get(var.basetype); @@ -2679,7 +2863,7 @@ void CompilerHLSL::emit_buffer_block(const SPIRVariable &var) type_name = is_readonly ? "ByteAddressBuffer" : is_interlocked ? "RasterizerOrderedByteAddressBuffer" : "RWByteAddressBuffer"; add_resource_name(var.self); - statement(is_coherent ? "globallycoherent " : "", type_name, " ", to_name(var.self), type_to_array_glsl(type), + statement(is_coherent ? "globallycoherent " : "", type_name, " ", to_name(var.self), type_to_array_glsl(type, var.self), to_resource_binding(var), ";"); } else @@ -2766,7 +2950,7 @@ void CompilerHLSL::emit_buffer_block(const SPIRVariable &var) } emit_struct(get(type.self)); - statement("ConstantBuffer<", to_name(type.self), "> ", to_name(var.self), type_to_array_glsl(type), + statement("ConstantBuffer<", to_name(type.self), "> ", to_name(var.self), type_to_array_glsl(type, var.self), to_resource_binding(var), ";"); } } @@ -2902,6 +3086,8 @@ string CompilerHLSL::get_inner_entry_point_name() const return "frag_main"; else if (execution.model == ExecutionModelGLCompute) return "comp_main"; + else if (execution.model == ExecutionModelGeometry) + return "geom_main"; else if (execution.model == ExecutionModelMeshEXT) return "mesh_main"; else if (execution.model == ExecutionModelTaskEXT) @@ -2910,9 +3096,33 @@ string CompilerHLSL::get_inner_entry_point_name() const SPIRV_CROSS_THROW("Unsupported execution model."); } +uint32_t CompilerHLSL::input_vertices_from_execution_mode(SPIREntryPoint &execution) const +{ + uint32_t input_vertices = 1; + + if (execution.flags.get(ExecutionModeInputLines)) + input_vertices = 2; + else if (execution.flags.get(ExecutionModeInputLinesAdjacency)) + input_vertices = 4; + else if (execution.flags.get(ExecutionModeInputTrianglesAdjacency)) + input_vertices = 6; + else if (execution.flags.get(ExecutionModeTriangles)) + input_vertices = 3; + else if (execution.flags.get(ExecutionModeInputPoints)) + input_vertices = 1; + else + SPIRV_CROSS_THROW("Unsupported execution model."); + return input_vertices; +} + void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &return_flags) { - if (func.self != ir.default_entry_point) + // In library mode default_entry_point points at the first exported + // function; treat every export as a normal function rather than as the + // shader's entry point. + const bool is_entry_point = !ir.is_library_module && func.self == ir.default_entry_point; + + if (!is_entry_point) add_function_overload(func); // Avoid shadow declarations. @@ -2923,7 +3133,7 @@ void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret auto &type = get(func.return_type); if (type.array.empty()) { - decl += flags_to_qualifiers_glsl(type, return_flags); + decl += flags_to_qualifiers_glsl(type, 0, return_flags); decl += type_to_glsl(type); decl += " "; } @@ -2933,7 +3143,7 @@ void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret decl = "void "; } - if (func.self == ir.default_entry_point) + if (is_entry_point) { decl += get_inner_entry_point_name(); processing_entry_point = true; @@ -2952,7 +3162,7 @@ void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret out_argument += type_to_glsl(type); out_argument += " "; out_argument += "spvReturnValue"; - out_argument += type_to_array_glsl(type); + out_argument += type_to_array_glsl(type, 0); arglist.push_back(std::move(out_argument)); } @@ -2978,7 +3188,7 @@ void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret { // Manufacture automatic sampler arg for SampledImage texture arglist.push_back(join(is_depth_image(arg_type, arg.id) ? "SamplerComparisonState " : "SamplerState ", - to_sampler_expression(arg.id), type_to_array_glsl(arg_type))); + to_sampler_expression(arg.id), type_to_array_glsl(arg_type, arg.id))); } // Hold a pointer to the parameter so we can invalidate the readonly field if needed. @@ -3003,6 +3213,38 @@ void CompilerHLSL::emit_function_prototype(SPIRFunction &func, const Bitset &ret var->parameter = &arg; } + if ((func.self == ir.default_entry_point || func.emits_geometry) && + get_entry_point().model == ExecutionModelGeometry) + { + auto &execution = get_entry_point(); + + uint32_t input_vertices = input_vertices_from_execution_mode(execution); + + const char *prim; + if (execution.flags.get(ExecutionModeInputLinesAdjacency)) + prim = "lineadj"; + else if (execution.flags.get(ExecutionModeInputLines)) + prim = "line"; + else if (execution.flags.get(ExecutionModeInputTrianglesAdjacency)) + prim = "triangleadj"; + else if (execution.flags.get(ExecutionModeTriangles)) + prim = "triangle"; + else + prim = "point"; + + const char *stream_type; + if (execution.flags.get(ExecutionModeOutputPoints)) + stream_type = "PointStream"; + else if (execution.flags.get(ExecutionModeOutputLineStrip)) + stream_type = "LineStream"; + else + stream_type = "TriangleStream"; + + if (func.self == ir.default_entry_point) + arglist.push_back(join(prim, " SPIRV_Cross_Input stage_input[", input_vertices, "]")); + arglist.push_back(join("inout ", stream_type, " ", "geometry_stream")); + } + decl += merge(arglist); decl += ")"; statement(decl); @@ -3012,13 +3254,52 @@ void CompilerHLSL::emit_hlsl_entry_point() { SmallVector arguments; - if (require_input) + if (require_input && get_entry_point().model != ExecutionModelGeometry) arguments.push_back("SPIRV_Cross_Input stage_input"); auto &execution = get_entry_point(); + uint32_t input_vertices = 1; + switch (execution.model) { + case ExecutionModelGeometry: + { + input_vertices = input_vertices_from_execution_mode(execution); + + string prim; + if (execution.flags.get(ExecutionModeInputLinesAdjacency)) + prim = "lineadj"; + else if (execution.flags.get(ExecutionModeInputLines)) + prim = "line"; + else if (execution.flags.get(ExecutionModeInputTrianglesAdjacency)) + prim = "triangleadj"; + else if (execution.flags.get(ExecutionModeTriangles)) + prim = "triangle"; + else + prim = "point"; + + string stream_type; + if (execution.flags.get(ExecutionModeOutputPoints)) + { + stream_type = "PointStream"; + } + else if (execution.flags.get(ExecutionModeOutputLineStrip)) + { + stream_type = "LineStream"; + } + else + { + stream_type = "TriangleStream"; + } + + statement("[maxvertexcount(", execution.output_vertices, ")]"); + arguments.push_back(join(prim, " SPIRV_Cross_Input stage_input[", input_vertices, "]")); + if (active_input_builtins.get(BuiltInPrimitiveId)) + arguments.push_back("uint gl_PrimitiveID : SV_PrimitiveID"); + arguments.push_back(join("inout ", stream_type, " ", "geometry_stream")); + break; + } case ExecutionModelTaskEXT: case ExecutionModelMeshEXT: case ExecutionModelGLCompute: @@ -3118,6 +3399,17 @@ void CompilerHLSL::emit_hlsl_entry_point() auto builtin = builtin_to_glsl(static_cast(i), StorageClassInput); switch (static_cast(i)) { + case BuiltInPosition: + if (execution.model == ExecutionModelGeometry) + { + statement("for (int i = 0; i < ", input_vertices, "; i++)"); + begin_scope(); + statement(builtin, "[i] = stage_input[i].", builtin, ";"); + end_scope(); + } + else + statement(builtin, " = stage_input.", builtin, ";"); + break; case BuiltInFragCoord: // VPOS in D3D9 is sampled at integer locations, apply half-pixel offset to be consistent. // TODO: Do we need an option here? Any reason why a D3D9 shader would be used @@ -3136,23 +3428,39 @@ void CompilerHLSL::emit_hlsl_entry_point() case BuiltInVertexIndex: case BuiltInInstanceIndex: // D3D semantics are uint, but shader wants int. - if (hlsl_options.support_nonzero_base_vertex_base_instance) + if (hlsl_options.support_nonzero_base_vertex_base_instance || hlsl_options.shader_model >= 68) { - if (static_cast(i) == BuiltInInstanceIndex) - statement(builtin, " = int(stage_input.", builtin, ") + SPIRV_Cross_BaseInstance;"); + if (hlsl_options.shader_model >= 68) + { + if (static_cast(i) == BuiltInInstanceIndex) + statement(builtin, " = int(stage_input.", builtin, " + stage_input.gl_BaseInstanceARB);"); + else + statement(builtin, " = int(stage_input.", builtin, " + stage_input.gl_BaseVertexARB);"); + } else - statement(builtin, " = int(stage_input.", builtin, ") + SPIRV_Cross_BaseVertex;"); + { + if (static_cast(i) == BuiltInInstanceIndex) + statement(builtin, " = int(stage_input.", builtin, ") + SPIRV_Cross_BaseInstance;"); + else + statement(builtin, " = int(stage_input.", builtin, ") + SPIRV_Cross_BaseVertex;"); + } } else statement(builtin, " = int(stage_input.", builtin, ");"); break; case BuiltInBaseVertex: - statement(builtin, " = SPIRV_Cross_BaseVertex;"); + if (hlsl_options.shader_model >= 68) + statement(builtin, " = stage_input.gl_BaseVertexARB;"); + else + statement(builtin, " = SPIRV_Cross_BaseVertex;"); break; case BuiltInBaseInstance: - statement(builtin, " = SPIRV_Cross_BaseInstance;"); + if (hlsl_options.shader_model >= 68) + statement(builtin, " = stage_input.gl_BaseInstanceARB;"); + else + statement(builtin, " = SPIRV_Cross_BaseInstance;"); break; case BuiltInInstanceId: @@ -3171,6 +3479,30 @@ void CompilerHLSL::emit_hlsl_entry_point() case BuiltInHelperInvocation: break; + case BuiltInPrimitiveId: + if (execution.model == ExecutionModelGeometry) + { + // PrimitiveId is a separate function parameter for GS. + // The global is named gl_PrimitiveIDIn (GLSL convention). + statement(builtin, " = gl_PrimitiveID;"); + } + else + statement(builtin, " = stage_input.", builtin, ";"); + break; + + case BuiltInInvocationId: + if (execution.model == ExecutionModelTessellationControl) + { + // Copy from function parameter to global. + statement(builtin, " = uCPID;"); + } + else + { + // For geometry shaders, copy from struct as usual. + statement(builtin, " = stage_input[0].", builtin, ";"); + } + break; + case BuiltInSubgroupEqMask: // Emulate these ... // No 64-bit in HLSL, so have to do it in 32-bit and unroll. @@ -3259,20 +3591,34 @@ void CompilerHLSL::emit_hlsl_entry_point() if (var.storage != StorageClassInput) return; + bool is_hidden = is_hidden_io_variable(var); + bool need_matrix_unroll = var.storage == StorageClassInput && execution.model == ExecutionModelVertex; if (!var.remapped_variable && type.pointer && !is_builtin_variable(var) && - interface_variable_exists_in_entry_point(var.self)) + interface_variable_exists_in_entry_point(var.self) && !is_hidden) { if (block) { auto type_name = to_name(type.self); auto var_name = to_name(var.self); + bool is_per_vertex = has_decoration(var.self, DecorationPerVertexKHR); + uint32_t array_size = is_per_vertex ? to_array_size_literal(type) : 0; + for (uint32_t mbr_idx = 0; mbr_idx < uint32_t(type.member_types.size()); mbr_idx++) { auto mbr_name = to_member_name(type, mbr_idx); auto flat_name = join(type_name, "_", mbr_name); - statement(var_name, ".", mbr_name, " = stage_input.", flat_name, ";"); + + if (is_per_vertex) + { + for (uint32_t i = 0; i < array_size; i++) + statement(var_name, "[", i, "].", mbr_name, " = GetAttributeAtVertex(stage_input.", flat_name, ", ", i, ");"); + } + else + { + statement(var_name, ".", mbr_name, " = stage_input.", flat_name, ";"); + } } } else @@ -3285,20 +3631,32 @@ void CompilerHLSL::emit_hlsl_entry_point() for (uint32_t col = 0; col < mtype.columns; col++) statement(name, "[", col, "] = stage_input.", name, "_", col, ";"); } + else if (has_decoration(var.self, DecorationPerVertexKHR)) + { + uint32_t array_size = to_array_size_literal(type); + for (uint32_t i = 0; i < array_size; i++) + statement(name, "[", i, "]", " = GetAttributeAtVertex(stage_input.", name, ", ", i, ");"); + } else { - statement(name, " = stage_input.", name, ";"); + if (execution.model == ExecutionModelGeometry) + { + statement("for (int i = 0; i < ", input_vertices, "; i++)"); + begin_scope(); + statement(name, "[i] = stage_input[i].", name, ";"); + end_scope(); + } + else + statement(name, " = stage_input.", name, ";"); } } } }); // Run the shader. - if (execution.model == ExecutionModelVertex || - execution.model == ExecutionModelFragment || - execution.model == ExecutionModelGLCompute || - execution.model == ExecutionModelMeshEXT || - execution.model == ExecutionModelTaskEXT) + if (execution.model == ExecutionModelVertex || execution.model == ExecutionModelFragment || + execution.model == ExecutionModelGLCompute || execution.model == ExecutionModelMeshEXT || + execution.model == ExecutionModelGeometry || execution.model == ExecutionModelTaskEXT) { // For mesh shaders, we receive special arguments that we must pass down as function arguments. // HLSL does not support proper reference types for passing these IO blocks, @@ -3306,8 +3664,16 @@ void CompilerHLSL::emit_hlsl_entry_point() SmallVector arglist; auto &func = get(ir.default_entry_point); // The arguments are marked out, avoid detecting reads and emitting inout. + for (auto &arg : func.arguments) arglist.push_back(to_expression(arg.id, false)); + + if (execution.model == ExecutionModelGeometry) + { + arglist.push_back("stage_input"); + arglist.push_back("geometry_stream"); + } + statement(get_inner_entry_point_name(), "(", merge(arglist), ");"); } else @@ -3505,19 +3871,19 @@ void CompilerHLSL::emit_texture_op(const Instruction &i, bool sparse) uint32_t coord_components = 0; switch (imgtype.image.dim) { - case spv::Dim1D: + case Dim1D: coord_components = 1; break; - case spv::Dim2D: + case Dim2D: coord_components = 2; break; - case spv::Dim3D: + case Dim3D: coord_components = 3; break; - case spv::DimCube: + case DimCube: coord_components = 3; break; - case spv::DimBuffer: + case DimBuffer: coord_components = 1; break; default: @@ -3528,7 +3894,7 @@ void CompilerHLSL::emit_texture_op(const Instruction &i, bool sparse) if (dref) inherited_expressions.push_back(dref); - if (imgtype.image.arrayed) + if (imgtype.image.arrayed && op != OpImageQueryLod) coord_components++; uint32_t bias = 0; @@ -3591,7 +3957,7 @@ void CompilerHLSL::emit_texture_op(const Instruction &i, bool sparse) else { auto &imgformat = get(imgtype.image.type); - if (hlsl_options.shader_model < 67 && imgformat.basetype != SPIRType::Float) + if (hlsl_options.shader_model < 67 && imgformat.basetype != SPIRType::Float && !gather) { SPIRV_CROSS_THROW("Sampling non-float textures is not supported in HLSL SM < 6.7."); } @@ -3746,7 +4112,7 @@ void CompilerHLSL::emit_texture_op(const Instruction &i, bool sparse) { if (dref) { - if (imgtype.image.dim != spv::Dim1D && imgtype.image.dim != spv::Dim2D) + if (imgtype.image.dim != Dim1D && imgtype.image.dim != Dim2D) { SPIRV_CROSS_THROW( "Depth comparison is only supported for 1D and 2D textures in HLSL shader model 2/3."); @@ -4076,16 +4442,16 @@ void CompilerHLSL::emit_modern_uniform(const SPIRVariable &var) is_coherent = has_decoration(var.self, DecorationCoherent); statement(is_coherent ? "globallycoherent " : "", image_type_hlsl_modern(type, var.self), " ", - to_name(var.self), type_to_array_glsl(type), to_resource_binding(var), ";"); + to_name(var.self), type_to_array_glsl(type, var.self), to_resource_binding(var), ";"); if (type.basetype == SPIRType::SampledImage && type.image.dim != DimBuffer) { // For combined image samplers, also emit a combined image sampler. if (is_depth_image(type, var.self)) - statement("SamplerComparisonState ", to_sampler_expression(var.self), type_to_array_glsl(type), + statement("SamplerComparisonState ", to_sampler_expression(var.self), type_to_array_glsl(type, var.self), to_resource_binding_sampler(var), ";"); else - statement("SamplerState ", to_sampler_expression(var.self), type_to_array_glsl(type), + statement("SamplerState ", to_sampler_expression(var.self), type_to_array_glsl(type, var.self), to_resource_binding_sampler(var), ";"); } break; @@ -4093,10 +4459,10 @@ void CompilerHLSL::emit_modern_uniform(const SPIRVariable &var) case SPIRType::Sampler: if (comparison_ids.count(var.self)) - statement("SamplerComparisonState ", to_name(var.self), type_to_array_glsl(type), to_resource_binding(var), + statement("SamplerComparisonState ", to_name(var.self), type_to_array_glsl(type, var.self), to_resource_binding(var), ";"); else - statement("SamplerState ", to_name(var.self), type_to_array_glsl(type), to_resource_binding(var), ";"); + statement("SamplerState ", to_name(var.self), type_to_array_glsl(type, var.self), to_resource_binding(var), ";"); break; default: @@ -4134,6 +4500,14 @@ bool CompilerHLSL::emit_complex_bitcast(uint32_t, uint32_t, uint32_t) return false; } +void CompilerHLSL::append_global_func_args(const SPIRFunction &func, uint32_t index, SmallVector &arglist) +{ + CompilerGLSL::append_global_func_args(func, index, arglist); + + if (func.emits_geometry) + arglist.push_back("geometry_stream"); +} + string CompilerHLSL::bitcast_glsl_op(const SPIRType &out_type, const SPIRType &in_type) { if (out_type.basetype == SPIRType::UInt && in_type.basetype == SPIRType::Int) @@ -4448,6 +4822,18 @@ void CompilerHLSL::emit_glsl_op(uint32_t result_type, uint32_t id, uint32_t eop, CompilerGLSL::emit_glsl_op(result_type, id, eop, args, count); break; + case GLSLstd450NMin: + CompilerGLSL::emit_glsl_op(result_type, id, GLSLstd450FMin, args, count); + break; + + case GLSLstd450NMax: + CompilerGLSL::emit_glsl_op(result_type, id, GLSLstd450FMax, args, count); + break; + + case GLSLstd450NClamp: + CompilerGLSL::emit_glsl_op(result_type, id, GLSLstd450FClamp, args, count); + break; + default: CompilerGLSL::emit_glsl_op(result_type, id, eop, args, count); break; @@ -4693,13 +5079,13 @@ void CompilerHLSL::emit_load(const Instruction &instruction) { auto ops = stream(instruction); - auto *chain = maybe_get(ops[2]); + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t ptr = ops[2]; + + auto *chain = maybe_get(ptr); if (chain) { - uint32_t result_type = ops[0]; - uint32_t id = ops[1]; - uint32_t ptr = ops[2]; - auto &type = get(result_type); bool composite_load = !type.array.empty() || type.basetype == SPIRType::Struct; @@ -4737,7 +5123,36 @@ void CompilerHLSL::emit_load(const Instruction &instruction) } } else - CompilerGLSL::emit_instruction(instruction); + { + // Very special case where we cannot rely on IO lowering. + // Mesh shader clip/cull arrays ... Cursed. + auto &res_type = get(result_type); + if (get_execution_model() == ExecutionModelMeshEXT && + has_decoration(ptr, DecorationBuiltIn) && + (get_decoration(ptr, DecorationBuiltIn) == BuiltInClipDistance || + get_decoration(ptr, DecorationBuiltIn) == BuiltInCullDistance) && + is_array(res_type) && !is_array(get(res_type.parent_type)) && + to_array_size_literal(res_type) > 1) + { + track_expression_read(ptr); + string load_expr = "{ "; + uint32_t num_elements = to_array_size_literal(res_type); + for (uint32_t i = 0; i < num_elements; i++) + { + load_expr += join(to_expression(ptr), ".", index_to_swizzle(i)); + if (i + 1 < num_elements) + load_expr += ", "; + } + load_expr += " }"; + emit_op(result_type, id, load_expr, false); + register_read(id, ptr, false); + inherit_expression_dependencies(id, ptr); + } + else + { + CompilerGLSL::emit_instruction(instruction); + } + } } void CompilerHLSL::write_access_chain_array(const SPIRAccessChain &chain, uint32_t value, @@ -4827,7 +5242,8 @@ string CompilerHLSL::write_access_chain_value(uint32_t value, const SmallVector< { AccessChainMeta meta; ret = access_chain_internal(value, composite_chain.data(), uint32_t(composite_chain.size()), - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_LITERAL_MSB_FORCE_ID, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_LITERAL_MSB_FORCE_ID, &meta, + nullptr); } if (enclose) @@ -5132,7 +5548,7 @@ void CompilerHLSL::emit_access_chain(const Instruction &instruction) } } -void CompilerHLSL::emit_atomic(const uint32_t *ops, uint32_t length, spv::Op op) +void CompilerHLSL::emit_atomic(const uint32_t *ops, uint32_t length, Op op) { const char *atomic_op = nullptr; @@ -5507,6 +5923,7 @@ void CompilerHLSL::emit_instruction(const Instruction &instruction) { auto ops = stream(instruction); auto opcode = static_cast(instruction.op); + uint32_t length = instruction.length; #define HLSL_BOP(op) emit_binary_op(ops[0], ops[1], ops[2], ops[3], #op) #define HLSL_BOP_CAST(op, type) \ @@ -6481,6 +6898,79 @@ void CompilerHLSL::emit_instruction(const Instruction &instruction) statement("SetMeshOutputCounts(", to_unpacked_expression(ops[0]), ", ", to_unpacked_expression(ops[1]), ");"); break; } + case OpEmitVertex: + { + emit_geometry_stream_append(); + break; + } + case OpEndPrimitive: + { + statement("geometry_stream.RestartStrip();"); + break; + } + + case OpSDot: + case OpUDot: + case OpSUDot: + case OpSDotAccSat: + case OpUDotAccSat: + case OpSUDotAccSat: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + bool is_acc_sat = opcode == OpSDotAccSat || opcode == OpUDotAccSat || opcode == OpSUDotAccSat; + + if (length == (is_acc_sat ? 6 : 5)) + { + if (ops[length - 1] != PackedVectorFormatPackedVectorFormat4x8Bit) + SPIRV_CROSS_THROW("Only 4x8bit packing is supported."); + } + + // Don't bother with polyfills. Integer dot products that aren't full speed are worthless. + if (hlsl_options.shader_model < 64) + SPIRV_CROSS_THROW("Integer dot product requires SM 6.4."); + if (opcode == OpSUDotAccSat || opcode == OpSUDot) + SPIRV_CROSS_THROW("Mixed signed dot product not supported."); + if (expression_type(ops[2]).vecsize != 1) + SPIRV_CROSS_THROW("HLSL dot products must be 4x8bit packed."); + if (integer_width != 32) + SPIRV_CROSS_THROW("HLSL dot products must be 32-bit accumulator."); + + const char *intrinsic; + if (opcode == OpSDot || opcode == OpSDotAccSat) + intrinsic = "dot4add_i8packed"; + else + intrinsic = "dot4add_u8packed"; + + auto expr = join(intrinsic, "(", to_expression(ops[2]), ", ", to_expression(ops[3]), ", "); + + // HLSL only has the accumulating variant without saturation. + // We could implement saturation ourselves, but it negates the point of using it. + // Take the lazier approach and just implement it as-is. + // Saturation is extremely unlikely to come up for any reasonable i8 kernel. + if (is_acc_sat) + expr += to_expression(ops[4]) + " /* WARN: HLSL will not saturate */)"; + else + expr += "0)"; + + if (((opcode == OpSDot || opcode == OpSDotAccSat) && get (result_type).basetype != SPIRType::Int) || + ((opcode == OpUDot || opcode == OpUDotAccSat) && get (result_type).basetype != SPIRType::UInt)) + { + expr = join(type_to_glsl(get (result_type)), "(", expr, ")"); + } + + bool forward = should_forward(ops[2]) && should_forward(ops[3]); + if (is_acc_sat && forward) + forward = should_forward(ops[4]); + + emit_op(result_type, id, expr, forward); + inherit_expression_dependencies(id, ops[2]); + inherit_expression_dependencies(id, ops[3]); + if (is_acc_sat) + inherit_expression_dependencies(id, ops[4]); + break; + } + default: CompilerGLSL::emit_instruction(instruction); break; @@ -6681,8 +7171,10 @@ string CompilerHLSL::compile() backend.can_return_array = false; backend.nonuniform_qualifier = "NonUniformResourceIndex"; backend.support_case_fallthrough = false; + backend.requires_phi_undef_zero_init = true; backend.force_merged_mesh_block = get_execution_model() == ExecutionModelMeshEXT; backend.force_gl_in_out_block = backend.force_merged_mesh_block; + backend.supports_empty_struct = hlsl_options.shader_model <= 30; // SM 4.1 does not support precise for some reason. backend.support_precise_qualifier = hlsl_options.shader_model >= 50 || hlsl_options.shader_model == 40; @@ -6698,10 +7190,22 @@ string CompilerHLSL::compile() if (get_execution_model() == ExecutionModelMeshEXT) analyze_meshlet_writes(); + if (get_execution_model() == ExecutionModelGeometry) + discover_geometry_emitters(); + // Subpass input needs SV_Position. if (need_subpass_input) active_input_builtins.set(BuiltInFragCoord); + // Need to offset by BaseVertex/BaseInstance in SM 6.8+. + if (hlsl_options.shader_model >= 68) + { + if (active_input_builtins.get(BuiltInVertexIndex)) + active_input_builtins.set(BuiltInBaseVertex); + if (active_input_builtins.get(BuiltInInstanceIndex)) + active_input_builtins.set(BuiltInBaseInstance); + } + uint32_t pass_count = 0; do { @@ -6713,14 +7217,27 @@ string CompilerHLSL::compile() emit_header(); emit_resources(); - emit_function(get(ir.default_entry_point), Bitset()); - emit_hlsl_entry_point(); + if (ir.is_library_module) + { + // Emit each exported function as a normal free function. + // emit_function recursively emits callees, so internal helpers + // are picked up too. + for (auto export_id : ir.library_exported_functions) + emit_function(get(export_id), Bitset()); + } + else + { + emit_function(get(ir.default_entry_point), Bitset()); + emit_hlsl_entry_point(); + } pass_count++; } while (is_forcing_recompilation()); // Entry point in HLSL is always main() for the time being. - get_entry_point().name = "main"; + // Skip the rename for library modules; their exports keep their declared names. + if (!ir.is_library_module) + get_entry_point().name = "main"; return buffer.str(); } @@ -6786,12 +7303,36 @@ bool CompilerHLSL::is_hlsl_force_storage_buffer_as_uav(ID id) const return true; } - const uint32_t desc_set = get_decoration(id, spv::DecorationDescriptorSet); - const uint32_t binding = get_decoration(id, spv::DecorationBinding); + const uint32_t desc_set = get_decoration(id, DecorationDescriptorSet); + const uint32_t binding = get_decoration(id, DecorationBinding); return (force_uav_buffer_bindings.find({ desc_set, binding }) != force_uav_buffer_bindings.end()); } +bool CompilerHLSL::is_hidden_io_variable(const SPIRVariable &var) const +{ + if (!is_hidden_variable(var)) + return false; + + // It is too risky to remove stage IO variables that are linkable since it affects link compatibility. + // For vertex inputs and fragment outputs, it's less of a concern and we want reflection data + // to match reality. + + bool is_external_linkage = + (get_execution_model() == ExecutionModelVertex && var.storage == StorageClassInput) || + (get_execution_model() == ExecutionModelFragment && var.storage == StorageClassOutput); + + if (!is_external_linkage) + return false; + + // Unused output I/O variables might still be required to implement framebuffer fetch. + if (var.storage == StorageClassOutput && !is_legacy() && + location_is_framebuffer_fetch(get_decoration(var.self, DecorationLocation)) != 0) + return false; + + return true; +} + void CompilerHLSL::set_hlsl_force_storage_buffer_as_uav(uint32_t desc_set, uint32_t binding) { SetBindingPair pair = { desc_set, binding }; @@ -6807,7 +7348,55 @@ bool CompilerHLSL::is_user_type_structured(uint32_t id) const const std::string &user_type = get_decoration_string(id, DecorationUserTypeGOOGLE); return user_type.compare(0, 16, "structuredbuffer") == 0 || user_type.compare(0, 18, "rwstructuredbuffer") == 0 || + user_type.compare(0, 35, "globallycoherent rwstructuredbuffer") == 0 || user_type.compare(0, 33, "rasterizerorderedstructuredbuffer") == 0; } return false; } + +void CompilerHLSL::cast_to_variable_store(uint32_t target_id, std::string &expr, const SPIRType &expr_type) +{ + // Loading a full array of ClipDistance needs special consideration in mesh shaders + // since we cannot lower them by wrapping the variables in global statics. + // Fortunately, clip/cull is a proper vector in HLSL so we can lower with simple rvalue casts. + if (get_execution_model() != ExecutionModelMeshEXT || + !has_decoration(target_id, DecorationBuiltIn) || + !is_array(expr_type)) + { + CompilerGLSL::cast_to_variable_store(target_id, expr, expr_type); + return; + } + + auto builtin = BuiltIn(get_decoration(target_id, DecorationBuiltIn)); + if (builtin != BuiltInClipDistance && builtin != BuiltInCullDistance) + { + CompilerGLSL::cast_to_variable_store(target_id, expr, expr_type); + return; + } + + // Array of array means one thread is storing clip distance for all vertices. Nonsensical? + if (is_array(get(expr_type.parent_type))) + SPIRV_CROSS_THROW("Attempting to store all mesh vertices in one go. This is not supported."); + + uint32_t num_clip = to_array_size_literal(expr_type); + if (num_clip > 4) + SPIRV_CROSS_THROW("Number of clip or cull distances exceeds 4, this will not work with mesh shaders."); + + if (num_clip == 1) + { + // We already emit array here. + CompilerGLSL::cast_to_variable_store(target_id, expr, expr_type); + return; + } + + auto unrolled_expr = join("float", num_clip, "("); + for (uint32_t i = 0; i < num_clip; i++) + { + unrolled_expr += join(expr, "[", i, "]"); + if (i + 1 < num_clip) + unrolled_expr += ", "; + } + + unrolled_expr += ")"; + expr = std::move(unrolled_expr); +} diff --git a/src/libraries/spirv_cross/spirv_hlsl.hpp b/src/libraries/spirv_cross/spirv_hlsl.hpp index bec458c61..0d5181b9d 100644 --- a/src/libraries/spirv_cross/spirv_hlsl.hpp +++ b/src/libraries/spirv_cross/spirv_hlsl.hpp @@ -29,6 +29,7 @@ namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; // Interface which remaps vertex inputs to a fixed semantic name to make linking easier. struct HLSLVertexAttributeRemap { @@ -87,7 +88,7 @@ using HLSLBindingFlags = uint32_t; // For deeper control of push constants, set_root_constant_layouts() can be used instead. struct HLSLResourceBinding { - spv::ExecutionModel stage = spv::ExecutionModelMax; + ExecutionModel stage = ExecutionModelMax; uint32_t desc_set = 0; uint32_t binding = 0; @@ -150,6 +151,9 @@ public: // This relies on UserTypeGOOGLE to encode the buffer type either as "structuredbuffer" or "rwstructuredbuffer" // whereas the type can be extended with an optional subtype, e.g. "structuredbuffer:int". bool preserve_structured_buffers = false; + + // Use UserSemantic decoration info (if specified), otherwise use default mechanism (such as add_vertex_attribute_remap or TEXCOORD#). + bool user_semantic = false; }; explicit CompilerHLSL(std::vector spirv_) @@ -216,7 +220,7 @@ public: // is_hlsl_resource_binding_used() will return true after calling ::compile() if // the set/binding combination was used by the HLSL code. void add_hlsl_resource_binding(const HLSLResourceBinding &resource); - bool is_hlsl_resource_binding_used(spv::ExecutionModel model, uint32_t set, uint32_t binding) const; + bool is_hlsl_resource_binding_used(ExecutionModel model, uint32_t set, uint32_t binding) const; // Controls which storage buffer bindings will be forced to be declared as UAVs. void set_hlsl_force_storage_buffer_as_uav(uint32_t desc_set, uint32_t binding); @@ -231,6 +235,7 @@ private: std::string image_type_hlsl(const SPIRType &type, uint32_t id); std::string image_type_hlsl_modern(const SPIRType &type, uint32_t id); std::string image_type_hlsl_legacy(const SPIRType &type, uint32_t id); + uint32_t input_vertices_from_execution_mode(SPIREntryPoint &execution) const; void emit_function_prototype(SPIRFunction &func, const Bitset &return_flags) override; void emit_hlsl_entry_point(); void emit_header() override; @@ -254,11 +259,13 @@ private: void emit_specialization_constants_and_structs(); void emit_composite_constants(); void emit_fixup() override; - std::string builtin_to_glsl(spv::BuiltIn builtin, spv::StorageClass storage) override; + std::string builtin_to_glsl(BuiltIn builtin, StorageClass storage) override; std::string layout_for_member(const SPIRType &type, uint32_t index) override; std::string to_interpolation_qualifiers(const Bitset &flags) override; std::string bitcast_glsl_op(const SPIRType &result_type, const SPIRType &argument_type) override; bool emit_complex_bitcast(uint32_t result_type, uint32_t id, uint32_t op0) override; + void append_global_func_args(const SPIRFunction &func, uint32_t index, SmallVector &arglist) override; + std::string to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_t id) override; std::string to_sampler_expression(uint32_t id); std::string to_resource_binding(const SPIRVariable &var); @@ -278,7 +285,7 @@ private: const SmallVector &composite_chain); std::string write_access_chain_value(uint32_t value, const SmallVector &composite_chain, bool enclose); void emit_store(const Instruction &instruction); - void emit_atomic(const uint32_t *ops, uint32_t length, spv::Op op); + void emit_atomic(const uint32_t *ops, uint32_t length, Op op); void emit_subgroup_op(const Instruction &i) override; void emit_block_hints(const SPIRBlock &block) override; @@ -286,13 +293,15 @@ private: uint32_t base_offset = 0) override; void emit_rayquery_function(const char *commited, const char *candidate, const uint32_t *ops); void emit_mesh_tasks(SPIRBlock &block) override; + void emit_geometry_stream_append(); const char *to_storage_qualifiers_glsl(const SPIRVariable &var) override; void replace_illegal_names() override; - SPIRType::BaseType get_builtin_basetype(spv::BuiltIn builtin, SPIRType::BaseType default_type) override; + SPIRType::BaseType get_builtin_basetype(BuiltIn builtin, SPIRType::BaseType default_type) override; bool is_hlsl_force_storage_buffer_as_uav(ID id) const; + bool is_hidden_io_variable(const SPIRVariable &var) const; Options hlsl_options; @@ -375,7 +384,7 @@ private: uint32_t type_to_consumed_locations(const SPIRType &type) const; - std::string to_semantic(uint32_t location, spv::ExecutionModel em, spv::StorageClass sc); + std::string to_semantic(uint32_t location, ExecutionModel em, StorageClass sc); uint32_t num_workgroups_builtin = 0; HLSLBindingFlags resource_binding_flags = 0; @@ -408,6 +417,8 @@ private: std::vector composite_selection_workaround_types; std::string get_inner_entry_point_name() const; + + void cast_to_variable_store(uint32_t target_id, std::string &expr, const SPIRType &expr_type) override; }; } // namespace SPIRV_CROSS_NAMESPACE diff --git a/src/libraries/spirv_cross/spirv_msl.cpp b/src/libraries/spirv_cross/spirv_msl.cpp index 200167a2e..c5a9d4332 100644 --- a/src/libraries/spirv_cross/spirv_msl.cpp +++ b/src/libraries/spirv_cross/spirv_msl.cpp @@ -28,7 +28,7 @@ #include #include -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; using namespace std; @@ -124,7 +124,7 @@ void CompilerMSL::add_msl_resource_binding(const MSLResourceBinding &binding) void CompilerMSL::add_dynamic_buffer(uint32_t desc_set, uint32_t binding, uint32_t index) { SetBindingPair pair = { desc_set, binding }; - buffers_requiring_dynamic_offset[pair] = { index, 0 }; + buffers_requiring_dynamic_offset[pair] = { index, 0, "" }; } void CompilerMSL::add_inline_uniform_block(uint32_t desc_set, uint32_t binding) @@ -164,7 +164,7 @@ bool CompilerMSL::is_msl_shader_output_used(uint32_t location) location_outputs_in_use_fallback.count(location) == 0; } -uint32_t CompilerMSL::get_automatic_builtin_input_location(spv::BuiltIn builtin) const +uint32_t CompilerMSL::get_automatic_builtin_input_location(BuiltIn builtin) const { auto itr = builtin_to_automatic_input_location.find(builtin); if (itr == builtin_to_automatic_input_location.end()) @@ -173,7 +173,7 @@ uint32_t CompilerMSL::get_automatic_builtin_input_location(spv::BuiltIn builtin) return itr->second; } -uint32_t CompilerMSL::get_automatic_builtin_output_location(spv::BuiltIn builtin) const +uint32_t CompilerMSL::get_automatic_builtin_output_location(BuiltIn builtin) const { auto itr = builtin_to_automatic_output_location.find(builtin); if (itr == builtin_to_automatic_output_location.end()) @@ -201,13 +201,20 @@ bool CompilerMSL::is_var_runtime_size_array(const SPIRVariable &var) const uint32_t CompilerMSL::get_resource_array_size(const SPIRType &type, uint32_t id) const { uint32_t array_size = to_array_size_literal(type); - if (array_size) + + if (id == 0) return array_size; - StageSetBinding tuple = { get_entry_point().model, get_decoration(id, DecorationDescriptorSet), + // If we have argument buffers, we need to honor the ABI by using the correct array size + // from the layout. Only use shader declared size if we're not using argument buffers. + uint32_t desc_set = get_decoration(id, DecorationDescriptorSet); + if (!descriptor_set_is_argument_buffer(desc_set) && array_size) + return array_size; + + StageSetBinding tuple = { get_entry_point().model, desc_set, get_decoration(id, DecorationBinding) }; auto itr = resource_bindings.find(tuple); - return itr != end(resource_bindings) ? itr->second.first.count : 0; + return itr != end(resource_bindings) ? itr->second.first.count : array_size; } uint32_t CompilerMSL::get_automatic_msl_resource_binding(uint32_t id) const @@ -235,7 +242,7 @@ void CompilerMSL::set_fragment_output_components(uint32_t location, uint32_t com fragment_output_components[location] = components; } -bool CompilerMSL::builtin_translates_to_nonarray(spv::BuiltIn builtin) const +bool CompilerMSL::builtin_translates_to_nonarray(BuiltIn builtin) const { return (builtin == BuiltInSampleMask); } @@ -265,13 +272,22 @@ void CompilerMSL::build_implicit_builtins() (active_input_builtins.get(BuiltInVertexId) || active_input_builtins.get(BuiltInVertexIndex) || active_input_builtins.get(BuiltInBaseVertex) || active_input_builtins.get(BuiltInInstanceId) || active_input_builtins.get(BuiltInInstanceIndex) || active_input_builtins.get(BuiltInBaseInstance)); - bool need_local_invocation_index = msl_options.emulate_subgroups && active_input_builtins.get(BuiltInSubgroupId); + bool need_local_invocation_index = + (msl_options.emulate_subgroups && active_input_builtins.get(BuiltInSubgroupId)) || is_mesh_shader() || + needs_workgroup_zero_init || needs_local_invocation_index; bool need_workgroup_size = msl_options.emulate_subgroups && active_input_builtins.get(BuiltInNumSubgroups); + bool force_frag_depth_passthrough = + get_execution_model() == ExecutionModelFragment && !uses_explicit_early_fragment_test() && need_subpass_input && + msl_options.enable_frag_depth_builtin && msl_options.input_attachment_is_ds_attachment; + needs_point_size_output = + msl_options.enable_point_size_builtin && msl_options.enable_point_size_default && + entry_point_is_vertex(); if (need_subpass_input || need_sample_pos || need_subgroup_mask || need_vertex_params || need_tesc_params || need_tese_params || need_multiview || need_dispatch_base || need_vertex_base_params || need_grid_params || needs_sample_id || needs_subgroup_invocation_id || needs_subgroup_size || needs_helper_invocation || - has_additional_fixed_sample_mask() || need_local_invocation_index || need_workgroup_size) + has_additional_fixed_sample_mask() || need_local_invocation_index || need_workgroup_size || + force_frag_depth_passthrough || needs_point_size_output || is_mesh_shader()) { bool has_frag_coord = false; bool has_sample_id = false; @@ -288,6 +304,8 @@ void CompilerMSL::build_implicit_builtins() bool has_helper_invocation = false; bool has_local_invocation_index = false; bool has_workgroup_size = false; + bool has_frag_depth = false; + bool has_point_size = false; uint32_t workgroup_id_type = 0; ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { @@ -295,6 +313,22 @@ void CompilerMSL::build_implicit_builtins() return; if (!interface_variable_exists_in_entry_point(var.self)) return; + + auto &type = this->get(var.basetype); + if (needs_point_size_output && has_decoration(type.self, DecorationBlock)) + { + const auto member_count = static_cast(type.member_types.size()); + for (uint32_t i = 0; i < member_count; i++) + { + if (get_member_decoration(type.self, i, DecorationBuiltIn) == BuiltInPointSize) + { + has_point_size = true; + active_output_builtins.set(BuiltInPointSize); + break; + } + } + } + if (!has_decoration(var.self, DecorationBuiltIn)) return; @@ -308,6 +342,26 @@ void CompilerMSL::build_implicit_builtins() mark_implicit_builtin(StorageClassOutput, BuiltInSampleMask, var.self); does_shader_write_sample_mask = true; } + + if (force_frag_depth_passthrough && builtin == BuiltInFragDepth) + { + builtin_frag_depth_id = var.self; + mark_implicit_builtin(StorageClassOutput, BuiltInFragDepth, var.self); + has_frag_depth = true; + } + } + + if (builtin == BuiltInPointSize) + { + has_point_size = true; + active_output_builtins.set(BuiltInPointSize); + } + + if (builtin == BuiltInPrimitivePointIndicesEXT || + builtin == BuiltInPrimitiveLineIndicesEXT || + builtin == BuiltInPrimitiveTriangleIndicesEXT) + { + builtin_mesh_primitive_indices_id = var.self; } if (var.storage != StorageClassInput) @@ -456,7 +510,7 @@ void CompilerMSL::build_implicit_builtins() has_local_invocation_index = true; } - if (need_workgroup_size && builtin == BuiltInLocalInvocationId) + if (need_workgroup_size && builtin == BuiltInWorkgroupSize) { builtin_workgroup_size_id = var.self; mark_implicit_builtin(StorageClassInput, BuiltInWorkgroupSize, var.self); @@ -630,28 +684,6 @@ void CompilerMSL::build_implicit_builtins() mark_implicit_builtin(StorageClassInput, BuiltInBaseInstance, var_id); } - if (need_multiview) - { - // Multiview shaders are not allowed to write to gl_Layer, ostensibly because - // it is implicitly written from gl_ViewIndex, but we have to do that explicitly. - // Note that we can't just abuse gl_ViewIndex for this purpose: it's an input, but - // gl_Layer is an output in vertex-pipeline shaders. - uint32_t type_ptr_out_id = ir.increase_bound_by(2); - SPIRType uint_type_ptr_out = get_uint_type(); - uint_type_ptr.op = OpTypePointer; - uint_type_ptr_out.pointer = true; - uint_type_ptr_out.pointer_depth++; - uint_type_ptr_out.parent_type = get_uint_type_id(); - uint_type_ptr_out.storage = StorageClassOutput; - auto &ptr_out_type = set(type_ptr_out_id, uint_type_ptr_out); - ptr_out_type.self = get_uint_type_id(); - uint32_t var_id = type_ptr_out_id + 1; - set(var_id, type_ptr_out_id, StorageClassOutput); - set_decoration(var_id, DecorationBuiltIn, BuiltInLayer); - builtin_layer_id = var_id; - mark_implicit_builtin(StorageClassOutput, BuiltInLayer, var_id); - } - if (need_multiview && !has_view_idx) { uint32_t var_id = ir.increase_bound_by(1); @@ -664,6 +696,28 @@ void CompilerMSL::build_implicit_builtins() } } + if (need_multiview) + { + // Multiview shaders are not allowed to write to gl_Layer, ostensibly because + // it is implicitly written from gl_ViewIndex, but we have to do that explicitly. + // Note that we can't just abuse gl_ViewIndex for this purpose: it's an input, but + // gl_Layer is an output in vertex-pipeline shaders. + uint32_t type_ptr_out_id = ir.increase_bound_by(2); + SPIRType uint_type_ptr_out = get_uint_type(); + uint_type_ptr_out.op = OpTypePointer; + uint_type_ptr_out.pointer = true; + uint_type_ptr_out.pointer_depth++; + uint_type_ptr_out.parent_type = get_uint_type_id(); + uint_type_ptr_out.storage = StorageClassOutput; + auto &ptr_out_type = set(type_ptr_out_id, uint_type_ptr_out); + ptr_out_type.self = get_uint_type_id(); + uint32_t var_id = type_ptr_out_id + 1; + set(var_id, type_ptr_out_id, StorageClassOutput); + set_decoration(var_id, DecorationBuiltIn, BuiltInLayer); + builtin_layer_id = var_id; + mark_implicit_builtin(StorageClassOutput, BuiltInLayer, var_id); + } + if ((need_tesc_params && (msl_options.multi_patch_workgroup || !has_invocation_id || !has_primitive_id)) || (need_tese_params && !has_primitive_id) || need_grid_params) { @@ -840,7 +894,7 @@ void CompilerMSL::build_implicit_builtins() set(type_id, bool_type); SPIRType bool_type_ptr_in = bool_type; - bool_type_ptr_in.op = spv::OpTypePointer; + bool_type_ptr_in.op = OpTypePointer; bool_type_ptr_in.pointer = true; bool_type_ptr_in.pointer_depth++; bool_type_ptr_in.parent_type = type_id; @@ -878,25 +932,113 @@ void CompilerMSL::build_implicit_builtins() if (need_workgroup_size && !has_workgroup_size) { - uint32_t offset = ir.increase_bound_by(2); - uint32_t type_ptr_id = offset; - uint32_t var_id = offset + 1; + auto &execution = get_entry_point(); + // First, check if the workgroup size _constant_ were defined. + // If it were, we don't need to do--in fact, shouldn't do--anything. + builtin_workgroup_size_id = execution.workgroup_size.constant; + if (builtin_workgroup_size_id == 0) + { + uint32_t var_id = ir.increase_bound_by(1); - // Create gl_WorkgroupSize. - uint32_t type_id = build_extended_vector_type(get_uint_type_id(), 3); - SPIRType uint_type_ptr = get(type_id); - uint_type_ptr.op = OpTypePointer; - uint_type_ptr.pointer = true; - uint_type_ptr.pointer_depth++; - uint_type_ptr.parent_type = type_id; - uint_type_ptr.storage = StorageClassInput; + // Create gl_WorkgroupSize. + uint32_t type_id = build_extended_vector_type(get_uint_type_id(), 3); + // If we have LocalSize or LocalSizeId, use those to define the workgroup size. + if (execution.flags.get(ExecutionModeLocalSizeId)) + { + const SPIRConstant *init[] = { &get(execution.workgroup_size.id_x), + &get(execution.workgroup_size.id_y), + &get(execution.workgroup_size.id_z) }; + bool specialized = init[0]->specialization || init[1]->specialization || init[2]->specialization; + set(var_id, type_id, init, 3, specialized); + execution.workgroup_size.constant = var_id; + } + else if (execution.flags.get(ExecutionModeLocalSize)) + { + uint32_t offset = ir.increase_bound_by(3); + const SPIRConstant *init[] = { + &set(offset, get_uint_type_id(), execution.workgroup_size.x, false), + &set(offset + 1, get_uint_type_id(), execution.workgroup_size.y, false), + &set(offset + 2, get_uint_type_id(), execution.workgroup_size.z, false) + }; + set(var_id, type_id, init, 3, false); + execution.workgroup_size.constant = var_id; + } + else + { + uint32_t type_ptr_id = ir.increase_bound_by(1); + SPIRType uint_type_ptr = get(type_id); + uint_type_ptr.op = OpTypePointer; + uint_type_ptr.pointer = true; + uint_type_ptr.pointer_depth++; + uint_type_ptr.parent_type = type_id; + uint_type_ptr.storage = StorageClassInput; - auto &ptr_type = set(type_ptr_id, uint_type_ptr); - ptr_type.self = type_id; - set(var_id, type_ptr_id, StorageClassInput); - set_decoration(var_id, DecorationBuiltIn, BuiltInWorkgroupSize); - builtin_workgroup_size_id = var_id; - mark_implicit_builtin(StorageClassInput, BuiltInWorkgroupSize, var_id); + auto &ptr_type = set(type_ptr_id, uint_type_ptr); + ptr_type.self = type_id; + set(var_id, type_ptr_id, StorageClassInput); + mark_implicit_builtin(StorageClassInput, BuiltInWorkgroupSize, var_id); + } + set_decoration(var_id, DecorationBuiltIn, BuiltInWorkgroupSize); + builtin_workgroup_size_id = var_id; + } + } + + if (!has_frag_depth && force_frag_depth_passthrough) + { + uint32_t offset = ir.increase_bound_by(3); + uint32_t type_id = offset; + uint32_t type_ptr_id = offset + 1; + uint32_t var_id = offset + 2; + + // Create gl_FragDepth + SPIRType float_type { OpTypeFloat }; + float_type.basetype = SPIRType::Float; + float_type.width = 32; + float_type.vecsize = 1; + set(type_id, float_type); + + SPIRType float_type_ptr_in = float_type; + float_type_ptr_in.op = OpTypePointer; + float_type_ptr_in.pointer = true; + float_type_ptr_in.pointer_depth++; + float_type_ptr_in.parent_type = type_id; + float_type_ptr_in.storage = StorageClassOutput; + + auto &ptr_in_type = set(type_ptr_id, float_type_ptr_in); + ptr_in_type.self = type_id; + set(var_id, type_ptr_id, StorageClassOutput); + set_decoration(var_id, DecorationBuiltIn, BuiltInFragDepth); + builtin_frag_depth_id = var_id; + mark_implicit_builtin(StorageClassOutput, BuiltInFragDepth, var_id); + active_output_builtins.set(BuiltInFragDepth); + } + + if (!has_point_size && needs_point_size_output) + { + uint32_t offset = ir.increase_bound_by(3); + uint32_t type_id = offset; + uint32_t type_ptr_id = offset + 1; + uint32_t var_id = offset + 2; + + // Create gl_PointSize + SPIRType float_type { OpTypeFloat }; + float_type.basetype = SPIRType::Float; + float_type.width = 32; + float_type.vecsize = 1; + set(type_id, float_type); + + SPIRType float_type_ptr_in = float_type; + float_type_ptr_in.op = OpTypePointer; + float_type_ptr_in.pointer = true; + float_type_ptr_in.pointer_depth++; + float_type_ptr_in.parent_type = type_id; + float_type_ptr_in.storage = StorageClassOutput; + + auto &ptr_in_type = set(type_ptr_id, float_type_ptr_in); + ptr_in_type.self = type_id; + set(var_id, type_ptr_id, StorageClassOutput); + set_decoration(var_id, DecorationBuiltIn, BuiltInPointSize); + mark_implicit_builtin(StorageClassOutput, BuiltInPointSize, var_id); } } @@ -945,9 +1087,22 @@ void CompilerMSL::build_implicit_builtins() dynamic_offsets_buffer_id = var_id; } + if (active_input_builtins.get(BuiltInDrawIndex)) + { + // This is always emulated. + uint32_t var_id = build_constant_uint_array_pointer(); + set_name(var_id, "spvDrawIndex"); + // This should never match anything. + set_decoration(var_id, DecorationDescriptorSet, ~(6u)); + set_decoration(var_id, DecorationBinding, msl_options.draw_id_buffer_index); + set_extended_decoration(var_id, SPIRVCrossDecorationResourceIndexPrimary, msl_options.draw_id_buffer_index); + draw_index_buffer_id = var_id; + } + // If we're returning a struct from a vertex-like entry point, we must return a position attribute. bool need_position = (get_execution_model() == ExecutionModelVertex || is_tese_shader()) && !capture_output_to_buffer && !get_is_rasterization_disabled() && + !msl_options.auto_disable_rasterization && !active_output_builtins.get(BuiltInPosition); if (need_position) @@ -984,6 +1139,10 @@ void CompilerMSL::build_implicit_builtins() }); need_position = has_output && !active_output_builtins.get(BuiltInPosition); } + else if (!active_output_builtins.get(BuiltInPosition) && msl_options.auto_disable_rasterization) + { + is_rasterization_disabled = true; + } if (need_position) { @@ -1012,12 +1171,59 @@ void CompilerMSL::build_implicit_builtins() set_decoration(var_id, DecorationBuiltIn, BuiltInPosition); mark_implicit_builtin(StorageClassOutput, BuiltInPosition, var_id); } + + if (is_mesh_shader()) + { + uint32_t offset = ir.increase_bound_by(2); + uint32_t type_ptr_id = offset; + uint32_t var_id = offset + 1; + + // Create variable to store meshlet size. + uint32_t type_id = build_extended_vector_type(get_uint_type_id(), 2); + SPIRType uint_type_ptr = get(type_id); + uint_type_ptr.op = OpTypePointer; + uint_type_ptr.pointer = true; + uint_type_ptr.pointer_depth++; + uint_type_ptr.parent_type = type_id; + uint_type_ptr.storage = StorageClassWorkgroup; + + auto &ptr_type = set(type_ptr_id, uint_type_ptr); + ptr_type.self = type_id; + set(var_id, type_ptr_id, StorageClassWorkgroup); + set_name(var_id, "spvMeshSizes"); + builtin_mesh_sizes_id = var_id; + } + + if (get_execution_model() == ExecutionModelTaskEXT) + { + uint32_t offset = ir.increase_bound_by(3); + uint32_t type_id = offset; + uint32_t type_ptr_id = offset + 1; + uint32_t var_id = offset + 2; + + SPIRType mesh_grid_type { OpTypeStruct }; + mesh_grid_type.basetype = SPIRType::MeshGridProperties; + set(type_id, mesh_grid_type); + + SPIRType mesh_grid_type_ptr = mesh_grid_type; + mesh_grid_type_ptr.op = OpTypePointer; + mesh_grid_type_ptr.pointer = true; + mesh_grid_type_ptr.pointer_depth++; + mesh_grid_type_ptr.parent_type = type_id; + mesh_grid_type_ptr.storage = StorageClassOutput; + + auto &ptr_in_type = set(type_ptr_id, mesh_grid_type_ptr); + ptr_in_type.self = type_id; + set(var_id, type_ptr_id, StorageClassOutput); + set_name(var_id, "spvMgp"); + builtin_task_grid_id = var_id; + } } // Checks if the specified builtin variable (e.g. gl_InstanceIndex) is marked as active. // If not, it marks it as active and forces a recompilation. // This might be used when the optimization of inactive builtins was too optimistic (e.g. when "spvOut" is emitted). -void CompilerMSL::ensure_builtin(spv::StorageClass storage, spv::BuiltIn builtin) +void CompilerMSL::ensure_builtin(StorageClass storage, BuiltIn builtin) { Bitset *active_builtins = nullptr; switch (storage) @@ -1300,69 +1506,53 @@ void CompilerMSL::emit_entry_point_declarations() // Emit dynamic buffers here. for (auto &dynamic_buffer : buffers_requiring_dynamic_offset) { - if (!dynamic_buffer.second.second) + if (!dynamic_buffer.second.var_id) { // Could happen if no buffer was used at requested binding point. continue; } - const auto &var = get(dynamic_buffer.second.second); + const auto &var = get(dynamic_buffer.second.var_id); uint32_t var_id = var.self; const auto &type = get_variable_data_type(var); + + add_local_variable_name(var.self); string name = to_name(var.self); + uint32_t desc_set = get_decoration(var.self, DecorationDescriptorSet); uint32_t arg_id = argument_buffer_ids[desc_set]; - uint32_t base_index = dynamic_buffer.second.first; + uint32_t base_index = dynamic_buffer.second.base_index; - if (!type.array.empty()) + if (is_array(type)) { - // This is complicated, because we need to support arrays of arrays. - // And it's even worse if the outermost dimension is a runtime array, because now - // all this complicated goop has to go into the shader itself. (FIXME) - if (!type.array[type.array.size() - 1]) - SPIRV_CROSS_THROW("Runtime arrays with dynamic offsets are not supported yet."); - else + is_using_builtin_array = true; + statement(get_variable_address_space(var), " ", type_to_glsl(type), "* ", to_restrict(var_id, true), name, + type_to_array_glsl(type, var_id), " ="); + + uint32_t array_size = get_resource_array_size(type, var_id); + if (array_size == 0) + SPIRV_CROSS_THROW("Size of runtime array with dynamic offset could not be determined from resource bindings."); + + begin_scope(); + + for (uint32_t i = 0; i < array_size; i++) { - is_using_builtin_array = true; - statement(get_argument_address_space(var), " ", type_to_glsl(type), "* ", to_restrict(var_id, true), name, - type_to_array_glsl(type), " ="); - - uint32_t dim = uint32_t(type.array.size()); - uint32_t j = 0; - for (SmallVector indices(type.array.size()); - indices[type.array.size() - 1] < to_array_size_literal(type); j++) - { - while (dim > 0) - { - begin_scope(); - --dim; - } - - string arrays; - for (uint32_t i = uint32_t(type.array.size()); i; --i) - arrays += join("[", indices[i - 1], "]"); - statement("(", get_argument_address_space(var), " ", type_to_glsl(type), "* ", - to_restrict(var_id, false), ")((", get_argument_address_space(var), " char* ", - to_restrict(var_id, false), ")", to_name(arg_id), ".", ensure_valid_name(name, "m"), - arrays, " + ", to_name(dynamic_offsets_buffer_id), "[", base_index + j, "]),"); - - while (++indices[dim] >= to_array_size_literal(type, dim) && dim < type.array.size() - 1) - { - end_scope(","); - indices[dim++] = 0; - } - } - end_scope_decl(); - statement_no_indent(""); - is_using_builtin_array = false; + statement("(", get_variable_address_space(var), " ", type_to_glsl(type), "* ", + to_restrict(var_id, false), ")((", get_variable_address_space(var), " char* ", + to_restrict(var_id, false), ")", to_name(arg_id), ".", dynamic_buffer.second.mbr_name, + "[", i, "]", " + ", to_name(dynamic_offsets_buffer_id), "[", base_index + i, "]),"); } + + end_scope_decl(); + statement_no_indent(""); + is_using_builtin_array = false; } else { - statement(get_argument_address_space(var), " auto& ", to_restrict(var_id, true), name, " = *(", - get_argument_address_space(var), " ", type_to_glsl(type), "* ", to_restrict(var_id, false), ")((", - get_argument_address_space(var), " char* ", to_restrict(var_id, false), ")", to_name(arg_id), ".", - ensure_valid_name(name, "m"), " + ", to_name(dynamic_offsets_buffer_id), "[", base_index, "]);"); + statement(get_variable_address_space(var), " auto& ", to_restrict(var_id, true), name, " = *(", + get_variable_address_space(var), " ", type_to_glsl(type), "* ", to_restrict(var_id, false), ")((", + get_variable_address_space(var), " char* ", to_restrict(var_id, false), ")", to_name(arg_id), ".", + dynamic_buffer.second.mbr_name, " + ", to_name(dynamic_offsets_buffer_id), "[", base_index, "]);"); } } @@ -1372,27 +1562,52 @@ void CompilerMSL::emit_entry_point_declarations() const auto &var = *arg; const auto &type = get_variable_data_type(var); const auto &buffer_type = get_variable_element_type(var); + + // This has already been added as a resource name. const string name = to_name(var.self); + if (is_var_runtime_size_array(var)) { if (msl_options.argument_buffers_tier < Options::ArgumentBuffersTier::Tier2) { SPIRV_CROSS_THROW("Unsized array of descriptors requires argument buffer tier 2"); } + + string resource_name; + + if (descriptor_set_is_argument_buffer(get_decoration(var.self, DecorationDescriptorSet))) + { + resource_name = ir.meta[var.self].decoration.qualified_alias; + } + else + { + bool is_aliased = std::find_if(buffer_aliases_discrete.begin(), buffer_aliases_discrete.end(), + [&](uint32_t id) { return var.self == id; }) != buffer_aliases_discrete.end(); + + uint32_t desc_set = get_decoration(var.self, DecorationDescriptorSet); + uint32_t desc_binding = get_decoration(var.self, DecorationBinding); + + if (is_aliased) + resource_name = join("spvBufferAliasSet", desc_set, "Binding", desc_binding); + else + resource_name = join("spvDescriptorSet", desc_set, "Binding", desc_binding); + } + switch (type.basetype) { case SPIRType::Image: case SPIRType::Sampler: case SPIRType::AccelerationStructure: - statement("spvDescriptorArray<", type_to_glsl(buffer_type), "> ", name, " {", name, "_};"); + statement("spvDescriptorArray<", type_to_glsl(buffer_type, var.self), "> ", name, " {", resource_name, "};"); break; case SPIRType::SampledImage: - statement("spvDescriptorArray<", type_to_glsl(buffer_type), "> ", name, " {", name, "_};"); - statement("spvDescriptorArray ", name, "Smplr {", name, "Smplr_};"); + statement("spvDescriptorArray<", type_to_glsl(buffer_type, var.self), "> ", name, " {", resource_name, "};"); + // Unsupported with argument buffer for now. + statement("spvDescriptorArray ", name, "Smplr {", resource_name, "Smplr};"); break; case SPIRType::Struct: - statement("spvDescriptorArray<", get_argument_address_space(var), " ", type_to_glsl(buffer_type), "*> ", - name, " {", name, "_};"); + statement("spvDescriptorArray<", get_variable_address_space(var), " ", type_to_glsl(buffer_type), "*> ", + name, " {", resource_name, "};"); break; default: break; @@ -1402,7 +1617,7 @@ void CompilerMSL::emit_entry_point_declarations() else if (!type.array.empty() && type.basetype == SPIRType::Struct) { // Emit only buffer arrays here. - statement(get_argument_address_space(var), " ", type_to_glsl(buffer_type), "* ", + statement(get_variable_address_space(var), " ", type_to_glsl(buffer_type), "* ", to_restrict(var.self, true), name, "[] ="); begin_scope(); uint32_t array_size = get_resource_array_size(type, var.self); @@ -1420,8 +1635,14 @@ void CompilerMSL::emit_entry_point_declarations() for (auto &var_id : buffer_aliases_discrete) { const auto &var = get(var_id); + + // We already declare this alias in a different way. + if (is_var_runtime_size_array(var)) + continue; + const auto &type = get_variable_data_type(var); - auto addr_space = get_argument_address_space(var); + auto addr_space = get_variable_address_space(var); + // This resource name has already been added. auto name = to_name(var_id); uint32_t desc_set = get_decoration(var_id, DecorationDescriptorSet); @@ -1435,36 +1656,6 @@ void CompilerMSL::emit_entry_point_declarations() // Discrete descriptors are processed in entry point emission every compiler iteration. buffer_aliases_discrete.clear(); - for (auto &var_pair : buffer_aliases_argument) - { - uint32_t var_id = var_pair.first; - uint32_t alias_id = var_pair.second; - - const auto &var = get(var_id); - const auto &type = get_variable_data_type(var); - auto addr_space = get_argument_address_space(var); - - if (type.array.empty()) - { - statement(addr_space, " auto& ", to_restrict(var_id, true), to_name(var_id), " = (", addr_space, " ", - type_to_glsl(type), "&)", ir.meta[alias_id].decoration.qualified_alias, ";"); - } - else - { - const char *desc_addr_space = descriptor_address_space(var_id, var.storage, "thread"); - - // Esoteric type cast. Reference to array of pointers. - // Auto here defers to UBO or SSBO. The address space of the reference needs to refer to the - // address space of the argument buffer itself, which is usually constant, but can be const device for - // large argument buffers. - is_using_builtin_array = true; - statement(desc_addr_space, " auto& ", to_restrict(var_id, true), to_name(var_id), " = (", addr_space, " ", - type_to_glsl(type), "* ", desc_addr_space, " (&)", - type_to_array_glsl(type), ")", ir.meta[alias_id].decoration.qualified_alias, ";"); - is_using_builtin_array = false; - } - } - // Emit disabled fragment outputs. std::sort(disabled_frag_outputs.begin(), disabled_frag_outputs.end()); for (uint32_t var_id : disabled_frag_outputs) @@ -1474,6 +1665,10 @@ void CompilerMSL::emit_entry_point_declarations() statement(CompilerGLSL::variable_decl(var), ";"); var.deferred_declaration = false; } + + // Holds SetMeshOutputsEXT information. Threadgroup since first thread wins. + if (processing_entry_point && is_mesh_shader()) + statement("threadgroup uint2 spvMeshSizes;"); } string CompilerMSL::compile() @@ -1497,6 +1692,7 @@ string CompilerMSL::compile() backend.basic_int16_type = "short"; backend.basic_uint16_type = "ushort"; backend.boolean_mix_function = "select"; + backend.printf_function = "os_log_default.log"; backend.swizzle_is_function = false; backend.shared_is_implied = false; backend.use_initializer_list = true; @@ -1509,6 +1705,8 @@ string CompilerMSL::compile() backend.native_pointers = true; backend.nonuniform_qualifier = ""; backend.support_small_type_sampling_result = true; + backend.force_merged_mesh_block = false; + backend.force_gl_in_out_block = false; backend.supports_empty_struct = true; backend.support_64bit_switch = true; backend.boolean_in_struct_remapped_type = SPIRType::Short; @@ -1520,10 +1718,14 @@ string CompilerMSL::compile() backend.array_is_value_type_in_buffer_blocks = false; backend.support_pointer_to_pointer = true; backend.implicit_c_integer_promotion_rules = true; + backend.supports_spec_constant_array_size = false; capture_output_to_buffer = msl_options.capture_output_to_buffer; is_rasterization_disabled = msl_options.disable_rasterization || capture_output_to_buffer; + if (is_mesh_shader() && !get_entry_point().flags.get(ExecutionModeOutputPoints)) + msl_options.enable_point_size_builtin = false; + // Initialize array here rather than constructor, MSVC 2013 workaround. for (auto &id : next_metal_resource_ids) id = 0; @@ -1531,6 +1733,11 @@ string CompilerMSL::compile() fixup_anonymous_struct_names(); fixup_type_alias(); replace_illegal_names(); + if (get_execution_model() == ExecutionModelMeshEXT) + { + // Emit proxy entry-point for the sake of copy-pass + emit_mesh_entry_point(); + } sync_entry_point_aliases_and_names(); build_function_control_flow_graphs_and_analyze(); @@ -1538,14 +1745,16 @@ string CompilerMSL::compile() analyze_image_and_sampler_usage(); analyze_sampled_image_usage(); analyze_interlocked_resource_usage(); + analyze_workgroup_variables(); preprocess_op_codes(); build_implicit_builtins(); - if (needs_manual_helper_invocation_updates() && - (active_input_builtins.get(BuiltInHelperInvocation) || needs_helper_invocation)) + if (needs_manual_helper_invocation_updates() && needs_helper_invocation) { - string discard_expr = - join(builtin_to_glsl(BuiltInHelperInvocation, StorageClassInput), " = true, discard_fragment()"); + string builtin_helper_invocation = builtin_to_glsl(BuiltInHelperInvocation, StorageClassInput); + string discard_expr = join(builtin_helper_invocation, " = true, discard_fragment()"); + if (msl_options.force_fragment_with_side_effects_execution) + discard_expr = join("!", builtin_helper_invocation, " ? (", discard_expr, ") : (void)0"); backend.discard_literal = discard_expr; backend.demote_literal = discard_expr; } @@ -1569,19 +1778,32 @@ string CompilerMSL::compile() add_active_interface_variable(view_mask_buffer_id); if (dynamic_offsets_buffer_id) add_active_interface_variable(dynamic_offsets_buffer_id); + if (draw_index_buffer_id) + add_active_interface_variable(draw_index_buffer_id); if (builtin_layer_id) add_active_interface_variable(builtin_layer_id); if (builtin_dispatch_base_id && !msl_options.supports_msl_version(1, 2)) add_active_interface_variable(builtin_dispatch_base_id); if (builtin_sample_mask_id) add_active_interface_variable(builtin_sample_mask_id); + if (builtin_frag_depth_id) + add_active_interface_variable(builtin_frag_depth_id); // Create structs to hold input, output and uniform variables. // Do output first to ensure out. is declared at top of entry function. qual_pos_var_name = ""; - stage_out_var_id = add_interface_block(StorageClassOutput); - patch_stage_out_var_id = add_interface_block(StorageClassOutput, true); - stage_in_var_id = add_interface_block(StorageClassInput); + qual_viewport_idx_var_name = ""; + if (is_mesh_shader()) + { + fixup_implicit_builtin_block_names(get_execution_model()); + } + else + { + stage_out_var_id = add_interface_block(StorageClassOutput); + patch_stage_out_var_id = add_interface_block(StorageClassOutput, true); + stage_in_var_id = add_interface_block(StorageClassInput); + } + if (is_tese_shader()) patch_stage_in_var_id = add_interface_block(StorageClassInput, true); @@ -1590,6 +1812,12 @@ string CompilerMSL::compile() if (is_tessellation_shader()) stage_in_ptr_var_id = add_interface_block_pointer(stage_in_var_id, StorageClassInput); + if (is_mesh_shader()) + { + mesh_out_per_vertex = add_meshlet_block(false); + mesh_out_per_primitive = add_meshlet_block(true); + } + // Metal vertex functions that define no output must disable rasterization and return void. if (!stage_out_var_id) is_rasterization_disabled = true; @@ -1654,7 +1882,7 @@ void CompilerMSL::preprocess_op_codes() if (preproc.uses_atomics) { add_header_line("#include "); - add_pragma_line("#pragma clang diagnostic ignored \"-Wunused-variable\""); + add_pragma_line("#pragma clang diagnostic ignored \"-Wunused-variable\"", false); } // Before MSL 2.1 (2.2 for textures), Metal vertex functions that write to @@ -1663,6 +1891,9 @@ void CompilerMSL::preprocess_op_codes() (preproc.uses_image_write && !msl_options.supports_msl_version(2, 2))) is_rasterization_disabled = true; + // FIXME: This currently does not consider BDA side effects, so we cannot deduce const device for BDA. + has_descriptor_side_effects_buffer = preproc.uses_buffer_write; + // Tessellation control shaders are run as compute functions in Metal, and so // must capture their output to a buffer. if (is_tesc_shader() || (get_execution_model() == ExecutionModelVertex && msl_options.vertex_for_tessellation)) @@ -1671,6 +1902,8 @@ void CompilerMSL::preprocess_op_codes() capture_output_to_buffer = true; } + if (preproc.needs_local_invocation_index) + needs_local_invocation_index = true; if (preproc.needs_subgroup_invocation_id) needs_subgroup_invocation_id = true; if (preproc.needs_subgroup_size) @@ -1682,7 +1915,7 @@ void CompilerMSL::preprocess_op_codes() (is_sample_rate() && (active_input_builtins.get(BuiltInFragCoord) || (need_subpass_input_ms && !msl_options.use_framebuffer_fetch_subpasses)))) needs_sample_id = true; - if (preproc.needs_helper_invocation) + if (preproc.needs_helper_invocation || active_input_builtins.get(BuiltInHelperInvocation)) needs_helper_invocation = true; // OpKill is removed by the parser, so we need to identify those by inspecting @@ -1694,14 +1927,15 @@ void CompilerMSL::preprocess_op_codes() // Fragment shaders that both write to storage resources and discard fragments // need checks on the writes, to work around Metal allowing these writes despite - // the fragment being dead. - if (msl_options.check_discarded_frag_stores && preproc.uses_discard && - (preproc.uses_buffer_write || preproc.uses_image_write)) + // the fragment being dead. We also require to force Metal to execute fragment + // shaders instead of being prematurely discarded. + if (preproc.uses_discard && (preproc.uses_buffer_write || preproc.uses_image_write)) { - frag_shader_needs_discard_checks = true; - needs_helper_invocation = true; + bool should_enable = (msl_options.check_discarded_frag_stores || msl_options.force_fragment_with_side_effects_execution); + frag_shader_needs_discard_checks |= msl_options.check_discarded_frag_stores; + needs_helper_invocation |= should_enable; // Fragment discard store checks imply manual HelperInvocation updates. - msl_options.manual_helper_invocation_updates = true; + msl_options.manual_helper_invocation_updates |= should_enable; } if (is_intersection_query()) @@ -1711,6 +1945,13 @@ void CompilerMSL::preprocess_op_codes() add_header_line("using namespace metal::raytracing;"); add_header_line("#endif"); } + + if (preproc.uses_cooperative_matrix) + { + if (!msl_options.supports_msl_version(3, 1)) + SPIRV_CROSS_THROW("Cooperative matrices require MSL 3.1 or later."); + add_header_line("#include "); + } } // Move the Private and Workgroup global variables to the entry function. @@ -1723,12 +1964,18 @@ void CompilerMSL::localize_global_variables() { uint32_t v_id = *iter; auto &var = get(v_id); - if (var.storage == StorageClassPrivate || var.storage == StorageClassWorkgroup) + if (var.storage == StorageClassPrivate || var.storage == StorageClassWorkgroup || + var.storage == StorageClassTaskPayloadWorkgroupEXT) { if (!variable_is_lut(var)) entry_func.add_local_variable(v_id); iter = global_variables.erase(iter); } + else if (var.storage == StorageClassOutput && is_mesh_shader()) + { + entry_func.add_local_variable(v_id); + iter = global_variables.erase(iter); + } else iter++; } @@ -1872,8 +2119,13 @@ void CompilerMSL::extract_global_variables_from_function(uint32_t func_id, std:: { uint32_t base_id = ops[0]; if (global_var_ids.find(base_id) != global_var_ids.end()) + { added_arg_ids.insert(base_id); + if (msl_options.input_attachment_is_ds_attachment && base_id == builtin_frag_depth_id) + writes_to_depth = true; + } + uint32_t rvalue_id = ops[1]; if (global_var_ids.find(rvalue_id) != global_var_ids.end()) added_arg_ids.insert(rvalue_id); @@ -1930,10 +2182,14 @@ void CompilerMSL::extract_global_variables_from_function(uint32_t func_id, std:: // When using the pointer, we need to know which variable it is actually loaded from. uint32_t base_id = ops[2]; auto *var = maybe_get_backing_variable(base_id); - if (var && atomic_image_vars_emulated.count(var->self)) + if (var) { - if (!get(var->basetype).array.empty()) - SPIRV_CROSS_THROW("Cannot emulate array of storage images with atomics. Use MSL 3.1 for native support."); + if (atomic_image_vars_emulated.count(var->self) && + !get(var->basetype).array.empty()) + { + SPIRV_CROSS_THROW( + "Cannot emulate array of storage images with atomics. Use MSL 3.1 for native support."); + } if (global_var_ids.find(base_id) != global_var_ids.end()) added_arg_ids.insert(base_id); @@ -2008,9 +2264,38 @@ void CompilerMSL::extract_global_variables_from_function(uint32_t func_id, std:: break; } + case OpGroupNonUniformRotateKHR: + { + // Add the correct invocation ID for calculating clustered rotate case. + if (i.length > 5) + added_arg_ids.insert(static_cast(evaluate_constant_u32(ops[2])) == ScopeSubgroup + ? builtin_subgroup_invocation_id_id : builtin_local_invocation_index_id); + break; + } + + case OpGroupNonUniformFAdd: + case OpGroupNonUniformFMul: + case OpGroupNonUniformFMin: + case OpGroupNonUniformFMax: + case OpGroupNonUniformIAdd: + case OpGroupNonUniformIMul: + case OpGroupNonUniformSMin: + case OpGroupNonUniformSMax: + case OpGroupNonUniformUMin: + case OpGroupNonUniformUMax: + case OpGroupNonUniformBitwiseAnd: + case OpGroupNonUniformBitwiseOr: + case OpGroupNonUniformBitwiseXor: + case OpGroupNonUniformLogicalAnd: + case OpGroupNonUniformLogicalOr: + case OpGroupNonUniformLogicalXor: + if ((get_execution_model() != ExecutionModelFragment || msl_options.supports_msl_version(2, 2)) && + ops[3] == GroupOperationClusteredReduce) + added_arg_ids.insert(builtin_subgroup_invocation_id_id); + break; + case OpDemoteToHelperInvocation: - if (needs_manual_helper_invocation_updates() && - (active_input_builtins.get(BuiltInHelperInvocation) || needs_helper_invocation)) + if (needs_manual_helper_invocation_updates() && needs_helper_invocation) added_arg_ids.insert(builtin_helper_invocation_id); break; @@ -2058,18 +2343,30 @@ void CompilerMSL::extract_global_variables_from_function(uint32_t func_id, std:: break; } + case OpSetMeshOutputsEXT: + { + if (builtin_local_invocation_index_id != 0) + added_arg_ids.insert(builtin_local_invocation_index_id); + if (builtin_mesh_sizes_id != 0) + added_arg_ids.insert(builtin_mesh_sizes_id); + break; + } + default: break; } if (needs_manual_helper_invocation_updates() && b.terminator == SPIRBlock::Kill && - (active_input_builtins.get(BuiltInHelperInvocation) || needs_helper_invocation)) + needs_helper_invocation) added_arg_ids.insert(builtin_helper_invocation_id); // TODO: Add all other operations which can affect memory. // We should consider a more unified system here to reduce boiler-plate. // This kind of analysis is done in several places ... } + + if (b.terminator == SPIRBlock::EmitMeshTasks && builtin_task_grid_id != 0) + added_arg_ids.insert(builtin_task_grid_id); } function_global_vars[func_id] = added_arg_ids; @@ -2159,6 +2456,24 @@ void CompilerMSL::extract_global_variables_from_function(uint32_t func_id, std:: if (is_tese_shader() && msl_options.raw_buffer_tese_input && var.storage == StorageClassInput) set_decoration(next_id, DecorationNonWritable); } + else if (is_builtin && is_mesh_shader()) + { + uint32_t next_id = ir.increase_bound_by(1); + func.add_parameter(type_id, next_id, true); + auto &v = set(next_id, type_id, StorageClassFunction, 0, arg_id); + v.storage = StorageClassWorkgroup; + + // Ensure the existing variable has a valid name and the new variable has all the same meta info + if (ir.meta[arg_id].decoration.builtin) + { + set_name(arg_id, builtin_to_glsl(bi_type, var.storage)); + } + else + { + set_name(arg_id, ensure_valid_name(to_name(arg_id), "v")); + } + ir.meta[next_id] = ir.meta[arg_id]; + } else if (is_builtin && has_decoration(p_type->self, DecorationBlock)) { // Get the pointee type @@ -2365,7 +2680,9 @@ uint32_t CompilerMSL::build_extended_vector_type(uint32_t type_id, uint32_t comp if (basetype != SPIRType::Unknown) type->basetype = basetype; type->self = new_type_id; - type->parent_type = type_id; + // We want parent type to point to the scalar type. + type->parent_type = is_scalar(*p_old_type) ? TypeID(p_old_type->self) : p_old_type->parent_type; + assert(is_scalar(get(type->parent_type))); type->array.clear(); type->array_size_literal.clear(); type->pointer = false; @@ -2409,7 +2726,7 @@ uint32_t CompilerMSL::build_msl_interpolant_type(uint32_t type_id, bool is_noper return new_type_id; } -bool CompilerMSL::add_component_variable_to_interface_block(spv::StorageClass storage, const std::string &ib_var_ref, +bool CompilerMSL::add_component_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRVariable &var, const SPIRType &type, InterfaceBlockMeta &meta) @@ -2670,6 +2987,8 @@ void CompilerMSL::add_plain_variable_to_interface_block(StorageClass storage, co set_member_decoration(ib_type.self, ib_mbr_idx, DecorationBuiltIn, builtin); if (builtin == BuiltInPosition && storage == StorageClassOutput) qual_pos_var_name = qual_var_name; + if (builtin == BuiltInViewportIndex && storage == StorageClassOutput) + qual_viewport_idx_var_name = qual_var_name; } // Copy interpolation decorations if needed @@ -2906,20 +3225,35 @@ void CompilerMSL::add_composite_member_variable_to_interface_block(StorageClass uint32_t mbr_idx, InterfaceBlockMeta &meta, const string &mbr_name_qual, const string &var_chain_qual, - uint32_t &location, uint32_t &var_mbr_idx) + uint32_t &location, uint32_t &var_mbr_idx, + const Bitset &interpolation_qual) { auto &entry_func = get(ir.default_entry_point); BuiltIn builtin = BuiltInMax; bool is_builtin = is_member_builtin(var_type, mbr_idx, &builtin); - bool is_flat = - has_member_decoration(var_type.self, mbr_idx, DecorationFlat) || has_decoration(var.self, DecorationFlat); - bool is_noperspective = has_member_decoration(var_type.self, mbr_idx, DecorationNoPerspective) || + bool is_flat = interpolation_qual.get(DecorationFlat) || + has_member_decoration(var_type.self, mbr_idx, DecorationFlat) || + has_decoration(var.self, DecorationFlat); + bool is_noperspective = interpolation_qual.get(DecorationNoPerspective) || + has_member_decoration(var_type.self, mbr_idx, DecorationNoPerspective) || has_decoration(var.self, DecorationNoPerspective); - bool is_centroid = has_member_decoration(var_type.self, mbr_idx, DecorationCentroid) || + bool is_centroid = interpolation_qual.get(DecorationCentroid) || + has_member_decoration(var_type.self, mbr_idx, DecorationCentroid) || has_decoration(var.self, DecorationCentroid); - bool is_sample = - has_member_decoration(var_type.self, mbr_idx, DecorationSample) || has_decoration(var.self, DecorationSample); + bool is_sample = interpolation_qual.get(DecorationSample) || + has_member_decoration(var_type.self, mbr_idx, DecorationSample) || + has_decoration(var.self, DecorationSample); + + Bitset inherited_qual; + if (is_flat) + inherited_qual.set(DecorationFlat); + if (is_noperspective) + inherited_qual.set(DecorationNoPerspective); + if (is_centroid) + inherited_qual.set(DecorationCentroid); + if (is_sample) + inherited_qual.set(DecorationSample); uint32_t mbr_type_id = var_type.member_types[mbr_idx]; auto &mbr_type = get(mbr_type_id); @@ -2983,7 +3317,7 @@ void CompilerMSL::add_composite_member_variable_to_interface_block(StorageClass add_composite_member_variable_to_interface_block(storage, ib_var_ref, ib_type, var, mbr_type, sub_mbr_idx, meta, mbr_name, var_chain, - location, var_mbr_idx); + location, var_mbr_idx, inherited_qual); // FIXME: Recursive structs and tessellation breaks here. var_mbr_idx++; } @@ -3004,41 +3338,62 @@ void CompilerMSL::add_composite_member_variable_to_interface_block(StorageClass string mbr_name = ensure_valid_name(append_member_name(mbr_name_qual, var_type, mbr_idx) + (mbr_is_indexable ? join("_", i) : ""), "m"); set_member_name(ib_type.self, ib_mbr_idx, mbr_name); + // The SPIRV location of interface variable, used to obtain the initial + // MSL location (the location variable) and interface matching + uint32_t ir_location = UINT32_MAX; + bool has_member_loc_decor = has_member_decoration(var_type.self, mbr_idx, DecorationLocation); + bool has_var_loc_decor = has_decoration(var.self, DecorationLocation); + uint32_t orig_vecsize = UINT32_MAX; + + // If we haven't established a location base yet, do so here. + if (location == UINT32_MAX) + { + if (has_member_loc_decor) + ir_location = get_member_decoration(var_type.self, mbr_idx, DecorationLocation); + else if (has_var_loc_decor) + ir_location = get_accumulated_member_location(var, mbr_idx, meta.strip_array); + else if (is_builtin) + { + if (is_tessellation_shader() && storage == StorageClassInput && inputs_by_builtin.count(builtin)) + ir_location = inputs_by_builtin[builtin].location; + else if (capture_output_to_buffer && storage == StorageClassOutput && outputs_by_builtin.count(builtin)) + ir_location = outputs_by_builtin[builtin].location; + } + } + // Once we determine the location of the first member within nested structures, // from a var of the topmost structure, the remaining flattened members of // the nested structures will have consecutive location values. At this point, // we've recursively tunnelled into structs, arrays, and matrices, and are // down to a single location for each member now. - if (!is_builtin && location != UINT32_MAX) + if (location == UINT32_MAX && ir_location != UINT32_MAX) + location = ir_location + i; + + if (storage == StorageClassInput && (has_member_loc_decor || has_var_loc_decor)) { - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, *usable_type, storage); - location++; + uint32_t component = 0; + uint32_t orig_mbr_type_id = usable_type->self; + + if (has_member_loc_decor) + component = get_member_decoration(var_type.self, mbr_idx, DecorationComponent); + + var.basetype = ensure_correct_input_type(var.basetype, location, component, 0, meta.strip_array); + mbr_type_id = ensure_correct_input_type(usable_type->self, location, component, 0, meta.strip_array); + + // For members of the composite interface block, we only change the interface block type + // when interface matching happens. In the meantime, we store the original vector size + // and insert a swizzle when loading from metal interface block (see fixup below) + if (mbr_type_id != orig_mbr_type_id) + orig_vecsize = get(orig_mbr_type_id).vecsize; + + if (storage == StorageClassInput && pull_model_inputs.count(var.self)) + ib_type.member_types[ib_mbr_idx] = build_msl_interpolant_type(mbr_type_id, is_noperspective); + else + ib_type.member_types[ib_mbr_idx] = mbr_type_id; } - else if (has_member_decoration(var_type.self, mbr_idx, DecorationLocation)) + + if ((!is_builtin && location != UINT32_MAX) || (is_builtin && ir_location != UINT32_MAX)) { - location = get_member_decoration(var_type.self, mbr_idx, DecorationLocation) + i; - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, *usable_type, storage); - location++; - } - else if (has_decoration(var.self, DecorationLocation)) - { - location = get_accumulated_member_location(var, mbr_idx, meta.strip_array) + i; - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, *usable_type, storage); - location++; - } - else if (is_builtin && is_tessellation_shader() && storage == StorageClassInput && inputs_by_builtin.count(builtin)) - { - location = inputs_by_builtin[builtin].location + i; - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, *usable_type, storage); - location++; - } - else if (is_builtin && capture_output_to_buffer && storage == StorageClassOutput && outputs_by_builtin.count(builtin)) - { - location = outputs_by_builtin[builtin].location + i; set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); mark_location_as_used_by_shader(location, *usable_type, storage); location++; @@ -3078,6 +3433,7 @@ void CompilerMSL::add_composite_member_variable_to_interface_block(StorageClass case StorageClassInput: entry_func.fixup_hooks_in.push_back([=, &var]() { string lerp_call; + string swizzle; if (pull_model_inputs.count(var.self)) { if (is_centroid) @@ -3087,7 +3443,9 @@ void CompilerMSL::add_composite_member_variable_to_interface_block(StorageClass else lerp_call = ".interpolate_at_center()"; } - statement(var_chain, " = ", ib_var_ref, ".", mbr_name, lerp_call, ";"); + if (orig_vecsize != UINT32_MAX) + swizzle = vector_swizzle(orig_vecsize, 0); + statement(var_chain, " = ", ib_var_ref, ".", mbr_name, lerp_call, swizzle, ";"); }); break; @@ -3155,6 +3513,55 @@ void CompilerMSL::add_plain_member_variable_to_interface_block(StorageClass stor qual_var_name += ".interpolate_at_center()"; } + // The SPIRV location of interface variable, used to obtain the initial + // MSL location (the location variable) and interface matching + uint32_t ir_location = UINT32_MAX; + bool has_member_loc_decor = has_member_decoration(var_type.self, mbr_idx, DecorationLocation); + bool has_var_loc_decor = has_decoration(var.self, DecorationLocation); + uint32_t orig_vecsize = UINT32_MAX; + + if (has_member_loc_decor) + ir_location = get_member_decoration(var_type.self, mbr_idx, DecorationLocation); + else if (has_var_loc_decor) + ir_location = get_accumulated_member_location(var, mbr_idx, meta.strip_array); + else if (is_builtin) + { + if (is_tessellation_shader() && storage == StorageClassInput && inputs_by_builtin.count(builtin)) + ir_location = inputs_by_builtin[builtin].location; + else if (capture_output_to_buffer && storage == StorageClassOutput && outputs_by_builtin.count(builtin)) + ir_location = outputs_by_builtin[builtin].location; + } + + // Once we determine the location of the first member within nested structures, + // from a var of the topmost structure, the remaining flattened members of + // the nested structures will have consecutive location values. At this point, + // we've recursively tunnelled into structs, arrays, and matrices, and are + // down to a single location for each member now. + if (location == UINT32_MAX && ir_location != UINT32_MAX) + location = ir_location; + + if (storage == StorageClassInput && (has_member_loc_decor || has_var_loc_decor)) + { + uint32_t component = 0; + uint32_t orig_mbr_type_id = mbr_type_id; + + if (has_member_loc_decor) + component = get_member_decoration(var_type.self, mbr_idx, DecorationComponent); + + mbr_type_id = ensure_correct_input_type(mbr_type_id, location, component, 0, meta.strip_array); + + // For members of the composite interface block, we only change the interface block type + // when interface matching happens. In the meantime, we store the original vector size + // and insert a swizzle when loading from metal interface block (see fixup below) + if (mbr_type_id != orig_mbr_type_id) + orig_vecsize = get(orig_mbr_type_id).vecsize; + + if (storage == StorageClassInput && pull_model_inputs.count(var.self)) + ib_type.member_types[ib_mbr_idx] = build_msl_interpolant_type(mbr_type_id, is_noperspective); + else + ib_type.member_types[ib_mbr_idx] = mbr_type_id; + } + bool flatten_stage_out = false; string var_chain = var_chain_qual + "." + to_member_name(var_type, mbr_idx); if (is_builtin && !meta.strip_array) @@ -3170,7 +3577,11 @@ void CompilerMSL::add_plain_member_variable_to_interface_block(StorageClass stor { case StorageClassInput: entry_func.fixup_hooks_in.push_back([=]() { - statement(var_chain, " = ", qual_var_name, ";"); + string swizzle; + // Insert swizzle for widened interface block vector from interface matching + if (orig_vecsize != UINT32_MAX) + swizzle = vector_swizzle(orig_vecsize, 0); + statement(var_chain, " = ", qual_var_name, swizzle, ";"); }); break; @@ -3186,64 +3597,12 @@ void CompilerMSL::add_plain_member_variable_to_interface_block(StorageClass stor } } - // Once we determine the location of the first member within nested structures, - // from a var of the topmost structure, the remaining flattened members of - // the nested structures will have consecutive location values. At this point, - // we've recursively tunnelled into structs, arrays, and matrices, and are - // down to a single location for each member now. - if (!is_builtin && location != UINT32_MAX) + if ((!is_builtin && location != UINT32_MAX) || (is_builtin && ir_location != UINT32_MAX)) { set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); mark_location_as_used_by_shader(location, get(mbr_type_id), storage); location += type_to_location_count(get(mbr_type_id)); } - else if (has_member_decoration(var_type.self, mbr_idx, DecorationLocation)) - { - location = get_member_decoration(var_type.self, mbr_idx, DecorationLocation); - uint32_t comp = get_member_decoration(var_type.self, mbr_idx, DecorationComponent); - if (storage == StorageClassInput) - { - mbr_type_id = ensure_correct_input_type(mbr_type_id, location, comp, 0, meta.strip_array); - var_type.member_types[mbr_idx] = mbr_type_id; - if (storage == StorageClassInput && pull_model_inputs.count(var.self)) - ib_type.member_types[ib_mbr_idx] = build_msl_interpolant_type(mbr_type_id, is_noperspective); - else - ib_type.member_types[ib_mbr_idx] = mbr_type_id; - } - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, get(mbr_type_id), storage); - location += type_to_location_count(get(mbr_type_id)); - } - else if (has_decoration(var.self, DecorationLocation)) - { - location = get_accumulated_member_location(var, mbr_idx, meta.strip_array); - if (storage == StorageClassInput) - { - mbr_type_id = ensure_correct_input_type(mbr_type_id, location, 0, 0, meta.strip_array); - var_type.member_types[mbr_idx] = mbr_type_id; - if (storage == StorageClassInput && pull_model_inputs.count(var.self)) - ib_type.member_types[ib_mbr_idx] = build_msl_interpolant_type(mbr_type_id, is_noperspective); - else - ib_type.member_types[ib_mbr_idx] = mbr_type_id; - } - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, get(mbr_type_id), storage); - location += type_to_location_count(get(mbr_type_id)); - } - else if (is_builtin && is_tessellation_shader() && storage == StorageClassInput && inputs_by_builtin.count(builtin)) - { - location = inputs_by_builtin[builtin].location; - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, get(mbr_type_id), storage); - location += type_to_location_count(get(mbr_type_id)); - } - else if (is_builtin && capture_output_to_buffer && storage == StorageClassOutput && outputs_by_builtin.count(builtin)) - { - location = outputs_by_builtin[builtin].location; - set_member_decoration(ib_type.self, ib_mbr_idx, DecorationLocation, location); - mark_location_as_used_by_shader(location, get(mbr_type_id), storage); - location += type_to_location_count(get(mbr_type_id)); - } // Copy the component location, if present. if (has_member_decoration(var_type.self, mbr_idx, DecorationComponent)) @@ -3258,6 +3617,8 @@ void CompilerMSL::add_plain_member_variable_to_interface_block(StorageClass stor set_member_decoration(ib_type.self, ib_mbr_idx, DecorationBuiltIn, builtin); if (builtin == BuiltInPosition && storage == StorageClassOutput) qual_pos_var_name = qual_var_name; + if (builtin == BuiltInViewportIndex && storage == StorageClassOutput) + qual_viewport_idx_var_name = qual_var_name; } const SPIRConstant *c = nullptr; @@ -3429,7 +3790,7 @@ void CompilerMSL::add_tess_level_input(const std::string &base_ref, const std::s } } -bool CompilerMSL::variable_storage_requires_stage_io(spv::StorageClass storage) const +bool CompilerMSL::variable_storage_requires_stage_io(StorageClass storage) const { if (storage == StorageClassOutput) return !capture_output_to_buffer; @@ -3470,7 +3831,7 @@ void CompilerMSL::emit_local_masked_variable(const SPIRVariable &masked_var, boo get_entry_point().output_vertices; statement("threadgroup ", type_to_glsl(type), " ", "spvStorage", to_name(masked_var.self), "[", max_num_instances, "]", - type_to_array_glsl(type), ";"); + type_to_array_glsl(type, 0), ";"); // Assign a threadgroup slice to each PrimitiveID. // We assume here that workgroup size is rounded to 32, @@ -3542,6 +3903,20 @@ void CompilerMSL::add_variable_to_interface_block(StorageClass storage, const st return; } + // Tesselation stages pass I/O via buffer content which may contain nested structs. + // Ensure the vector sizes of any nested struct members within these input variables match + // the vector sizes of the corresponding output variables from the previous pipeline stage. + // This adjustment is handled here instead of ensure_correct_input_type() in order to + // perform the necessary recursive processing. + if (storage == StorageClassInput && var_type.basetype == SPIRType::Struct && + ((is_tesc_shader() && msl_options.multi_patch_workgroup) || + (is_tese_shader() && msl_options.raw_buffer_tese_input)) && + has_decoration(var.self, DecorationLocation)) + { + uint32_t locn = get_decoration(var.self, DecorationLocation); + ensure_struct_members_valid_vecsizes(get_variable_data_type(var), locn); + } + if (storage == StorageClassInput && has_decoration(var.self, DecorationPerVertexKHR)) SPIRV_CROSS_THROW("PerVertexKHR decoration is not supported in MSL."); @@ -3669,7 +4044,7 @@ void CompilerMSL::add_variable_to_interface_block(StorageClass storage, const st add_composite_member_variable_to_interface_block(storage, ib_var_ref, ib_type, var, var_type, mbr_idx, meta, mbr_name_qual, var_chain_qual, - location, var_mbr_idx); + location, var_mbr_idx, {}); } else { @@ -3737,6 +4112,43 @@ void CompilerMSL::add_variable_to_interface_block(StorageClass storage, const st } } +// Recursively iterate into the input struct type, and adjust the vecsize +// of any nested members, based on location info provided through the API. +// The location parameter is modified recursively. +void CompilerMSL::ensure_struct_members_valid_vecsizes(SPIRType &struct_type, uint32_t &location) +{ + assert(struct_type.basetype == SPIRType::Struct); + + auto mbr_cnt = struct_type.member_types.size(); + for (size_t mbr_idx = 0; mbr_idx < mbr_cnt; mbr_idx++) + { + auto mbr_type_id = struct_type.member_types[mbr_idx]; + auto &mbr_type = get(mbr_type_id); + + if (mbr_type.basetype == SPIRType::Struct) + ensure_struct_members_valid_vecsizes(mbr_type, location); + else + { + auto p_va = inputs_by_location.find({location, 0}); + if (p_va != end(inputs_by_location) && p_va->second.vecsize > mbr_type.vecsize) + { + // Set a new member type into the struct type, and all its parent types. + auto new_mbr_type_id = build_extended_vector_type(mbr_type_id, p_va->second.vecsize); + for (auto *p_type = &struct_type; p_type; p_type = maybe_get(p_type->parent_type)) + p_type->member_types[mbr_idx] = new_mbr_type_id; + } + + // Calc location of next member + uint32_t loc_cnt = mbr_type.columns; + auto dim_cnt = mbr_type.array.size(); + for (uint32_t i = 0; i < dim_cnt; i++) + loc_cnt *= to_array_size_literal(mbr_type, i); + + location += loc_cnt; + } + } +} + // Fix up the mapping of variables to interface member indices, which is used to compile access chains // for per-vertex variables in a tessellation control shader. void CompilerMSL::fix_up_interface_member_indices(StorageClass storage, uint32_t ib_type_id) @@ -4041,8 +4453,9 @@ uint32_t CompilerMSL::add_interface_block(StorageClass storage, bool patch) // If the entry point should return the output struct, set the entry function // to return the output interface struct, otherwise to return nothing. // Watch out for the rare case where the terminator of the last entry point block is a - // Kill, instead of a Return. Based on SPIR-V's block-domination rules, we assume that - // any block that has a Kill will also have a terminating Return, except the last block. + // Kill or Unreachable, instead of a Return. Based on SPIR-V's block-domination rules, + // we assume that any block that has a Kill will also have a terminating Return, except + // the last block. // Indicate the output var requires early initialization. bool ep_should_return_output = !get_is_rasterization_disabled(); uint32_t rtn_id = ep_should_return_output ? ib_var_id : 0; @@ -4052,7 +4465,8 @@ uint32_t CompilerMSL::add_interface_block(StorageClass storage, bool patch) for (auto &blk_id : entry_func.blocks) { auto &blk = get(blk_id); - if (blk.terminator == SPIRBlock::Return || (blk.terminator == SPIRBlock::Kill && blk_id == entry_func.blocks.back())) + auto last_blk_return = blk.terminator == SPIRBlock::Kill || blk.terminator == SPIRBlock::Unreachable; + if (blk.terminator == SPIRBlock::Return || (last_blk_return && blk_id == entry_func.blocks.back())) blk.return_value = rtn_id; } vars_needing_early_declaration.push_back(ib_var_id); @@ -4426,6 +4840,42 @@ uint32_t CompilerMSL::add_interface_block_pointer(uint32_t ib_var_id, StorageCla return ib_ptr_var_id; } +uint32_t CompilerMSL::add_meshlet_block(bool per_primitive) +{ + // Accumulate the variables that should appear in the interface struct. + SmallVector vars; + + ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { + if (var.storage != StorageClassOutput || var.self == builtin_mesh_primitive_indices_id) + return; + if (is_per_primitive_variable(var) != per_primitive) + return; + vars.push_back(&var); + }); + + if (vars.empty()) + return 0; + + uint32_t next_id = ir.increase_bound_by(1); + auto &type = set(next_id, SPIRType(OpTypeStruct)); + type.basetype = SPIRType::Struct; + + InterfaceBlockMeta meta; + for (auto *p_var : vars) + { + meta.strip_array = true; + meta.allow_local_declaration = false; + add_variable_to_interface_block(StorageClassOutput, "", type, *p_var, meta); + } + + if (per_primitive) + set_name(type.self, "spvPerPrimitive"); + else + set_name(type.self, "spvPerVertex"); + + return next_id; +} + // Ensure that the type is compatible with the builtin. // If it is, simply return the given type ID. // Otherwise, create a new type, and return it's ID. @@ -4438,18 +4888,18 @@ uint32_t CompilerMSL::ensure_correct_builtin_type(uint32_t type_id, BuiltIn buil ((builtin == BuiltInLayer || builtin == BuiltInViewportIndex || builtin == BuiltInFragStencilRefEXT) && pointee_type.basetype != SPIRType::UInt)) { - uint32_t next_id = ir.increase_bound_by(type_is_pointer(type) ? 2 : 1); + uint32_t next_id = ir.increase_bound_by(is_pointer(type) ? 2 : 1); uint32_t base_type_id = next_id++; auto &base_type = set(base_type_id, OpTypeInt); base_type.basetype = SPIRType::UInt; base_type.width = 32; - if (!type_is_pointer(type)) + if (!is_pointer(type)) return base_type_id; uint32_t ptr_type_id = next_id++; auto &ptr_type = set(ptr_type_id, base_type); - ptr_type.op = spv::OpTypePointer; + ptr_type.op = OpTypePointer; ptr_type.pointer = true; ptr_type.pointer_depth++; ptr_type.storage = type.storage; @@ -4624,23 +5074,16 @@ void CompilerMSL::mark_scalar_layout_structs(const SPIRType &type) if (struct_needs_explicit_padding) { - msl_size = get_declared_struct_size_msl(*struct_type, true, true); + msl_size = get_declared_struct_size_msl(*struct_type); + + if (array_stride > msl_size) + { + set_decoration(struct_type->self, DecorationArrayStride, msl_size); + add_spv_func_and_recompile(SPVFuncImplPaddedArrayElement); + } + if (array_stride < msl_size) - { SPIRV_CROSS_THROW("Cannot express an array stride smaller than size of struct type."); - } - else - { - if (has_extended_decoration(struct_type->self, SPIRVCrossDecorationPaddingTarget)) - { - if (array_stride != - get_extended_decoration(struct_type->self, SPIRVCrossDecorationPaddingTarget)) - SPIRV_CROSS_THROW( - "A struct is used with different array strides. Cannot express this in MSL."); - } - else - set_extended_decoration(struct_type->self, SPIRVCrossDecorationPaddingTarget, array_stride); - } } } } @@ -4737,7 +5180,7 @@ bool CompilerMSL::validate_member_packing_rules_msl(const SPIRType &type, uint32 return false; } - if (!mbr_type.array.empty()) + if (is_array(mbr_type)) { // If we have an array type, array stride must match exactly with SPIR-V. @@ -4746,8 +5189,10 @@ bool CompilerMSL::validate_member_packing_rules_msl(const SPIRType &type, uint32 // If app tries to be cheeky and access the member out of bounds, this will not work, but this is the best we can do. // In OpAccessChain with logical memory models, access chains must be in-bounds in SPIR-V specification. bool relax_array_stride = mbr_type.array.back() == 1 && mbr_type.array_size_literal.back(); + bool is_plain_struct = !mbr_type.pointer && mbr_type.basetype == SPIRType::Struct; - if (!relax_array_stride) + // Array of struct is padded on-demand. + if (!relax_array_stride && !is_plain_struct) { uint32_t spirv_array_stride = type_struct_member_array_stride(type, index); uint32_t msl_array_stride = get_declared_struct_member_array_stride_msl(type, index); @@ -5325,16 +5770,44 @@ void CompilerMSL::emit_header() { // This particular line can be overridden during compilation, so make it a flag and not a pragma line. if (suppress_missing_prototypes) - statement("#pragma clang diagnostic ignored \"-Wmissing-prototypes\""); + add_pragma_line("#pragma clang diagnostic ignored \"-Wmissing-prototypes\"", false); + if (suppress_incompatible_pointer_types_discard_qualifiers) + add_pragma_line("#pragma clang diagnostic ignored \"-Wincompatible-pointer-types-discards-qualifiers\"", false); + + // Disable warning about "sometimes unitialized" when zero-initializing simple threadgroup variables + if (suppress_sometimes_unitialized) + add_pragma_line("#pragma clang diagnostic ignored \"-Wsometimes-uninitialized\"", false); // Disable warning about missing braces for array template to make arrays a value type if (spv_function_implementations.count(SPVFuncImplUnsafeArray) != 0) - statement("#pragma clang diagnostic ignored \"-Wmissing-braces\""); + add_pragma_line("#pragma clang diagnostic ignored \"-Wmissing-braces\"", false); + + // Floating point fast math compile declarations + if (msl_options.use_fast_math_pragmas && msl_options.supports_msl_version(3, 2)) + { + uint32_t contract_mask = FPFastMathModeAllowContractMask; + uint32_t relax_mask = (FPFastMathModeNSZMask | FPFastMathModeAllowRecipMask | FPFastMathModeAllowReassocMask); + uint32_t fast_mask = (relax_mask | FPFastMathModeNotNaNMask | FPFastMathModeNotInfMask); + + // FP math mode + uint32_t fp_flags = get_fp_fast_math_flags(true); + const char *math_mode = "safe"; + if ((fp_flags & fast_mask) == fast_mask) // Must have all flags + math_mode = "fast"; + else if ((fp_flags & relax_mask) == relax_mask) // Must have all flags + math_mode = "relaxed"; + + add_pragma_line(join("#pragma metal fp math_mode(", math_mode, ")"), false); + + // FP contraction + const char *contract_mode = ((fp_flags & contract_mask) == contract_mask) ? "fast" : "off"; + add_pragma_line(join("#pragma metal fp contract(", contract_mode, ")"), false); + } for (auto &pragma : pragma_lines) statement(pragma); - if (!pragma_lines.empty() || suppress_missing_prototypes) + if (!pragma_lines.empty()) statement(""); statement("#include "); @@ -5354,18 +5827,23 @@ void CompilerMSL::emit_header() statement(""); } -void CompilerMSL::add_pragma_line(const string &line) +void CompilerMSL::add_pragma_line(const string &line, bool recompile_on_unique) { - auto rslt = pragma_lines.insert(line); - if (rslt.second) - force_recompile(); + if (std::find(pragma_lines.begin(), pragma_lines.end(), line) == pragma_lines.end()) + { + pragma_lines.push_back(line); + if (recompile_on_unique) + force_recompile(); + } } void CompilerMSL::add_typedef_line(const string &line) { - auto rslt = typedef_lines.insert(line); - if (rslt.second) + if (std::find(typedef_lines.begin(), typedef_lines.end(), line) == typedef_lines.end()) + { + typedef_lines.push_back(line); force_recompile(); + } } // Template struct like spvUnsafeArray<> need to be declared *before* any resources are declared @@ -5416,6 +5894,19 @@ void CompilerMSL::emit_custom_templates() begin_scope(); statement("return elements[pos];"); end_scope(); + if (get_execution_model() == ExecutionModelMeshEXT || + get_execution_model() == ExecutionModelTaskEXT) + { + statement(""); + statement("object_data T& operator [] (size_t pos) object_data"); + begin_scope(); + statement("return elements[pos];"); + end_scope(); + statement("constexpr const object_data T& operator [] (size_t pos) const object_data"); + begin_scope(); + statement("return elements[pos];"); + end_scope(); + } end_scope_decl(); statement(""); break; @@ -5547,9 +6038,12 @@ void CompilerMSL::emit_custom_templates() // otherwise they will cause problems when linked together in a single Metallib. void CompilerMSL::emit_custom_functions() { - for (uint32_t i = kArrayCopyMultidimMax; i >= 2; i--) - if (spv_function_implementations.count(static_cast(SPVFuncImplArrayCopyMultidimBase + i))) - spv_function_implementations.insert(static_cast(SPVFuncImplArrayCopyMultidimBase + i - 1)); + // Use when outputting overloaded functions to cover different address spaces. + static const char *texture_addr_spaces[] = { "device", "constant", "thread" }; + static uint32_t texture_addr_space_count = sizeof(texture_addr_spaces) / sizeof(char*); + + if (spv_function_implementations.count(SPVFuncImplArrayCopyMultidim)) + spv_function_implementations.insert(SPVFuncImplArrayCopy); if (spv_function_implementations.count(SPVFuncImplDynamicImageSampler)) { @@ -5557,7 +6051,6 @@ void CompilerMSL::emit_custom_functions() if (!msl_options.supports_msl_version(2)) SPIRV_CROSS_THROW( "spvDynamicImageSampler requires default-constructible texture objects, which require MSL 2.0."); - spv_function_implementations.insert(SPVFuncImplForwardArgs); spv_function_implementations.insert(SPVFuncImplTextureSwizzle); if (msl_options.swizzle_texture_samples) spv_function_implementations.insert(SPVFuncImplGatherSwizzle); @@ -5571,16 +6064,22 @@ void CompilerMSL::emit_custom_functions() spv_function_implementations.insert(SPVFuncImplConvertYCbCrBT2020); } - for (uint32_t i = SPVFuncImplChromaReconstructNearest2Plane; - i <= SPVFuncImplChromaReconstructLinear420XMidpointYMidpoint3Plane; i++) - if (spv_function_implementations.count(static_cast(i))) - spv_function_implementations.insert(SPVFuncImplForwardArgs); + if (spv_function_implementations.count(SPVFuncImplGatherSwizzle) || + spv_function_implementations.count(SPVFuncImplGatherConstOffsets)) + { + spv_function_implementations.insert(SPVFuncImplGatherReturn); + } + + if (spv_function_implementations.count(SPVFuncImplGatherCompareSwizzle) || + spv_function_implementations.count(SPVFuncImplGatherCompareConstOffsets)) + { + spv_function_implementations.insert(SPVFuncImplGatherCompareReturn); + } if (spv_function_implementations.count(SPVFuncImplTextureSwizzle) || spv_function_implementations.count(SPVFuncImplGatherSwizzle) || spv_function_implementations.count(SPVFuncImplGatherCompareSwizzle)) { - spv_function_implementations.insert(SPVFuncImplForwardArgs); spv_function_implementations.insert(SPVFuncImplGetSwizzle); } @@ -5588,6 +6087,17 @@ void CompilerMSL::emit_custom_functions() { switch (spv_func) { + case SPVFuncImplSMod: + statement("// Implementation of signed integer mod accurate to SPIR-V specification"); + statement("template"); + statement("inline Tx spvSMod(Tx x, Ty y)"); + begin_scope(); + statement("Tx remainder = x - y * (x / y);"); + statement("return select(Tx(remainder + y), remainder, remainder == 0 || (x >= 0) == (y >= 0));"); + end_scope(); + statement(""); + break; + case SPVFuncImplMod: statement("// Implementation of the GLSL mod() function, which is slightly different than Metal fmod()"); statement("template"); @@ -5660,11 +6170,7 @@ void CompilerMSL::emit_custom_functions() break; case SPVFuncImplArrayCopy: - case SPVFuncImplArrayOfArrayCopy2Dim: - case SPVFuncImplArrayOfArrayCopy3Dim: - case SPVFuncImplArrayOfArrayCopy4Dim: - case SPVFuncImplArrayOfArrayCopy5Dim: - case SPVFuncImplArrayOfArrayCopy6Dim: + case SPVFuncImplArrayCopyMultidim: { // Unfortunately we cannot template on the address space, so combinatorial explosion it is. static const char *function_name_tags[] = { @@ -5685,40 +6191,115 @@ void CompilerMSL::emit_custom_functions() "device", "device", "device", "device", "thread", "threadgroup", }; + static const bool src_is_physical_with_mismatch[] = { + true, true, false, + false, false, false, + false, false, false, + false, true, true, + }; + + static const bool dst_is_physical_with_mismatch[] = { + false, false, false, + false, false, false, + false, false, true, + true, false, false, + }; + for (uint32_t variant = 0; variant < 12; variant++) { - uint8_t dimensions = spv_func - SPVFuncImplArrayCopyMultidimBase; - string tmp = "template" : ">"); + statement("inline void spvArrayCopy", function_name_tags[variant], "(", + dst_address_space[variant], " T (&dst)", dim, ", ", + src_address_space[variant], " T (&src)", dim, ")"); begin_scope(); - statement("for (uint i = 0; i < A; i++)"); + statement("for (uint i = 0; i < N; i++)"); begin_scope(); - - if (dimensions == 1) - statement("dst[i] = src[i];"); + if (is_multidim) + statement("spvArrayCopy", function_name_tags[variant], "(dst[i], src[i]);"); else - statement("spvArrayCopy", function_name_tags[variant], dimensions - 1, "(dst[i], src[i]);"); + statement("dst[i] = src[i];"); end_scope(); end_scope(); + + if (spv_function_implementations.count(SPVFuncImplArrayCopyExtendedSrc) && + src_is_physical_with_mismatch[variant]) + { + // 1st overload, src can be magic vector where dst is a scalar. + // Need reinterpret casts to be memory model correct. LLVM vectors are broken otherwise. + statement("template" : ">"); + statement("inline void spvArrayCopy", function_name_tags[variant], "(", + dst_address_space[variant], " T (&dst)", dim, ", ", + src_address_space[variant], " vec (&src)", dim, ")"); + begin_scope(); + statement("for (uint i = 0; i < N; i++)"); + begin_scope(); + if (is_multidim) + statement("spvArrayCopy", function_name_tags[variant], "(dst[i], src[i]);"); + else + statement("dst[i] = reinterpret_cast<", src_address_space[variant], " T &>(src[i]);"); + end_scope(); + end_scope(); + + statement(""); + + // 2nd overload, both are vectors, but need SFINAE magic to avoid ambiguous case. + statement("template" : ">"); + statement("inline enable_if_t spvArrayCopy", function_name_tags[variant], "(", + dst_address_space[variant], " vec (&dst)", dim, ", ", + src_address_space[variant], " vec (&src)", dim, ")"); + begin_scope(); + statement("for (uint i = 0; i < N; i++)"); + begin_scope(); + if (is_multidim) + statement("spvArrayCopy", function_name_tags[variant], "(dst[i], src[i]);"); + else + statement("dst[i] = reinterpret_cast<", src_address_space[variant], " vec &>(src[i]);"); + end_scope(); + end_scope(); + } + + if (spv_function_implementations.count(SPVFuncImplArrayCopyExtendedDst) && + dst_is_physical_with_mismatch[variant]) + { + // 1st overload, src can be magic vector where dst is a scalar. + // Need reinterpret casts to be memory model correct. LLVM vectors are broken otherwise. + statement("template" : ">"); + statement("inline void spvArrayCopy", function_name_tags[variant], "(", + dst_address_space[variant], " vec (&dst)", dim, ", ", + src_address_space[variant], " T (&src)", dim, ")"); + begin_scope(); + statement("for (uint i = 0; i < N; i++)"); + begin_scope(); + if (is_multidim) + statement("spvArrayCopy", function_name_tags[variant], "(dst[i], src[i]);"); + else + statement("reinterpret_cast<", dst_address_space[variant], " T &>(dst[i]) = src[i];"); + end_scope(); + end_scope(); + + statement(""); + + // 2nd overload, both are vectors, but need SFINAE magic to avoid ambiguous case. + statement("template" : ">"); + statement("inline enable_if_t spvArrayCopy", function_name_tags[variant], "(", + dst_address_space[variant], " vec (&dst)", dim, ", ", + src_address_space[variant], " vec (&src)", dim, ")"); + begin_scope(); + statement("for (uint i = 0; i < N; i++)"); + begin_scope(); + if (is_multidim) + statement("spvArrayCopy", function_name_tags[variant], "(dst[i], src[i]);"); + else + statement("reinterpret_cast<", dst_address_space[variant], " vec &>(dst[i]) = src[i];"); + end_scope(); + end_scope(); + } + statement(""); } break; @@ -6071,23 +6652,6 @@ void CompilerMSL::emit_custom_functions() statement(""); break; - case SPVFuncImplForwardArgs: - statement("template struct spvRemoveReference { typedef T type; };"); - statement("template struct spvRemoveReference { typedef T type; };"); - statement("template struct spvRemoveReference { typedef T type; };"); - statement("template inline constexpr thread T&& spvForward(thread typename " - "spvRemoveReference::type& x)"); - begin_scope(); - statement("return static_cast(x);"); - end_scope(); - statement("template inline constexpr thread T&& spvForward(thread typename " - "spvRemoveReference::type&& x)"); - begin_scope(); - statement("return static_cast(x);"); - end_scope(); - statement(""); - break; - case SPVFuncImplGetSwizzle: statement("enum class spvSwizzle : uint"); begin_scope(); @@ -6145,11 +6709,22 @@ void CompilerMSL::emit_custom_functions() statement(""); break; + case SPVFuncImplGatherReturn: + statement("template"); + statement("using spvGatherReturn = decltype(declval().gather(declval(), declval()...));"); + statement(""); + break; + + case SPVFuncImplGatherCompareReturn: + statement("template"); + statement("using spvGatherCompareReturn = decltype(declval().gather_compare(declval(), declval()...));"); + statement(""); + break; + case SPVFuncImplGatherSwizzle: statement("// Wrapper function that swizzles texture gathers."); - statement("template class Tex, " - "typename... Ts>"); - statement("inline vec spvGatherSwizzle(const thread Tex& t, sampler s, " + statement("template"); + statement("inline spvGatherReturn spvGatherSwizzle(const thread Tex& t, sampler s, " "uint sw, component c, Ts... params) METAL_CONST_ARG(c)"); begin_scope(); statement("if (sw)"); @@ -6159,17 +6734,17 @@ void CompilerMSL::emit_custom_functions() statement("case spvSwizzle::none:"); statement(" break;"); statement("case spvSwizzle::zero:"); - statement(" return vec(0, 0, 0, 0);"); + statement(" return spvGatherReturn(0, 0, 0, 0);"); statement("case spvSwizzle::one:"); - statement(" return vec(1, 1, 1, 1);"); + statement(" return spvGatherReturn(1, 1, 1, 1);"); statement("case spvSwizzle::red:"); - statement(" return t.gather(s, spvForward(params)..., component::x);"); + statement(" return t.gather(s, params..., component::x);"); statement("case spvSwizzle::green:"); - statement(" return t.gather(s, spvForward(params)..., component::y);"); + statement(" return t.gather(s, params..., component::y);"); statement("case spvSwizzle::blue:"); - statement(" return t.gather(s, spvForward(params)..., component::z);"); + statement(" return t.gather(s, params..., component::z);"); statement("case spvSwizzle::alpha:"); - statement(" return t.gather(s, spvForward(params)..., component::w);"); + statement(" return t.gather(s, params..., component::w);"); end_scope(); end_scope(); // texture::gather insists on its component parameter being a constant @@ -6177,13 +6752,13 @@ void CompilerMSL::emit_custom_functions() statement("switch (c)"); begin_scope(); statement("case component::x:"); - statement(" return t.gather(s, spvForward(params)..., component::x);"); + statement(" return t.gather(s, params..., component::x);"); statement("case component::y:"); - statement(" return t.gather(s, spvForward(params)..., component::y);"); + statement(" return t.gather(s, params..., component::y);"); statement("case component::z:"); - statement(" return t.gather(s, spvForward(params)..., component::z);"); + statement(" return t.gather(s, params..., component::z);"); statement("case component::w:"); - statement(" return t.gather(s, spvForward(params)..., component::w);"); + statement(" return t.gather(s, params..., component::w);"); end_scope(); end_scope(); statement(""); @@ -6191,10 +6766,8 @@ void CompilerMSL::emit_custom_functions() case SPVFuncImplGatherCompareSwizzle: statement("// Wrapper function that swizzles depth texture gathers."); - statement("template class Tex, " - "typename... Ts>"); - statement("inline vec spvGatherCompareSwizzle(const thread Tex& t, sampler " - "s, uint sw, Ts... params) "); + statement("template"); + statement("inline spvGatherCompareReturn spvGatherCompareSwizzle(const thread Tex& t, sampler s, uint sw, Ts... params)"); begin_scope(); statement("if (sw)"); begin_scope(); @@ -6207,16 +6780,73 @@ void CompilerMSL::emit_custom_functions() statement("case spvSwizzle::green:"); statement("case spvSwizzle::blue:"); statement("case spvSwizzle::alpha:"); - statement(" return vec(0, 0, 0, 0);"); + statement(" return spvGatherCompareReturn(0, 0, 0, 0);"); statement("case spvSwizzle::one:"); - statement(" return vec(1, 1, 1, 1);"); + statement(" return spvGatherCompareReturn(1, 1, 1, 1);"); end_scope(); end_scope(); - statement("return t.gather_compare(s, spvForward(params)...);"); + statement("return t.gather_compare(s, params...);"); end_scope(); statement(""); break; + case SPVFuncImplGatherConstOffsets: + // Because we are passing a texture reference, we have to output an overloaded version of this function for each address space. + for (uint32_t i = 0; i < texture_addr_space_count; i++) + { + statement("// Wrapper function that processes a ", texture_addr_spaces[i], " texture gather with a constant offset array."); + statement("template"); + statement("inline spvGatherReturn spvGatherConstOffsets(const ", texture_addr_spaces[i], " Tex& t, sampler s, " + "Toff coffsets, component c, Tp... params) METAL_CONST_ARG(c)"); + begin_scope(); + statement("spvGatherReturn rslts[4];"); + statement("for (uint i = 0; i < 4; i++)"); + begin_scope(); + statement("switch (c)"); + begin_scope(); + // Work around texture::gather() requiring its component parameter to be a constant expression + statement("case component::x:"); + statement(" rslts[i] = t.gather(s, params..., coffsets[i], component::x);"); + statement(" break;"); + statement("case component::y:"); + statement(" rslts[i] = t.gather(s, params..., coffsets[i], component::y);"); + statement(" break;"); + statement("case component::z:"); + statement(" rslts[i] = t.gather(s, params..., coffsets[i], component::z);"); + statement(" break;"); + statement("case component::w:"); + statement(" rslts[i] = t.gather(s, params..., coffsets[i], component::w);"); + statement(" break;"); + end_scope(); + end_scope(); + // Pull all values from the i0j0 component of each gather footprint + statement("return spvGatherReturn(rslts[0].w, rslts[1].w, rslts[2].w, rslts[3].w);"); + end_scope(); + statement(""); + } + break; + + case SPVFuncImplGatherCompareConstOffsets: + // Because we are passing a texture reference, we have to output an overloaded version of this function for each address space. + for (uint32_t i = 0; i < texture_addr_space_count; i++) + { + statement("// Wrapper function that processes a ", texture_addr_spaces[i], " texture gather with a constant offset array."); + statement("template"); + statement("inline spvGatherCompareReturn spvGatherCompareConstOffsets(const ", texture_addr_spaces[i], " Tex& t, sampler s, " + "Toff coffsets, Tp... params)"); + begin_scope(); + statement("spvGatherCompareReturn rslts[4];"); + statement("for (uint i = 0; i < 4; i++)"); + begin_scope(); + statement(" rslts[i] = t.gather_compare(s, params..., coffsets[i]);"); + end_scope(); + // Pull all values from the i0j0 component of each gather footprint + statement("return spvGatherCompareReturn(rslts[0].w, rslts[1].w, rslts[2].w, rslts[3].w);"); + end_scope(); + statement(""); + } + break; + case SPVFuncImplSubgroupBroadcast: // Metal doesn't allow broadcasting boolean values directly, but we can work around that by broadcasting // them as integers. @@ -6464,6 +7094,40 @@ void CompilerMSL::emit_custom_functions() statement("return (vec)simd_shuffle((vec)value, lane);"); end_scope(); statement(""); + + if (msl_options.supports_msl_version(2, 2)) + { + // Despite being a template in MSL, it does not support 64-bit shuffles. + // Unsure if there's a cleaner way to statically unroll based on vec<> template, but this will do. + statement("template<>"); + statement("inline ulong spvSubgroupShuffle(ulong value, ushort lane)"); + begin_scope(); + statement("return as_type(spvSubgroupShuffle(as_type(value), lane));"); + end_scope(); + statement(""); + statement("template<>"); + statement("inline ulong2 spvSubgroupShuffle(ulong2 value, ushort lane)"); + begin_scope(); + statement("return ulong2(spvSubgroupShuffle(value.x, lane), spvSubgroupShuffle(value.y, lane));"); + end_scope(); + statement(""); + statement("inline ulong3 spvSubgroupShuffle(ulong3 value, ushort lane)"); + begin_scope(); + statement("return ulong3(spvSubgroupShuffle(value.xy, lane), spvSubgroupShuffle(value.z, lane));"); + end_scope(); + statement(""); + statement("inline ulong4 spvSubgroupShuffle(ulong4 value, ushort lane)"); + begin_scope(); + statement("return ulong4(spvSubgroupShuffle(value.xy, lane), spvSubgroupShuffle(value.zw, lane));"); + end_scope(); + statement(""); + statement("template"); + statement("inline vec spvSubgroupShuffle(vec value, ushort lane)"); + begin_scope(); + statement("return vec(spvSubgroupShuffle(vec(value), lane));"); + end_scope(); + statement(""); + } break; case SPVFuncImplSubgroupShuffleXor: @@ -6556,6 +7220,135 @@ void CompilerMSL::emit_custom_functions() statement(""); break; + case SPVFuncImplSubgroupRotate: + statement("template"); + statement("inline T spvSubgroupRotate(T value, ushort delta)"); + begin_scope(); + if (msl_options.use_quadgroup_operation()) + statement("return quad_shuffle_rotate_down(value, delta);"); + else + statement("return simd_shuffle_rotate_down(value, delta);"); + end_scope(); + statement(""); + statement("template<>"); + statement("inline bool spvSubgroupRotate(bool value, ushort delta)"); + begin_scope(); + if (msl_options.use_quadgroup_operation()) + statement("return !!quad_shuffle_rotate_down((ushort)value, delta);"); + else + statement("return !!simd_shuffle_rotate_down((ushort)value, delta);"); + end_scope(); + statement(""); + statement("template"); + statement("inline vec spvSubgroupRotate(vec value, ushort delta)"); + begin_scope(); + if (msl_options.use_quadgroup_operation()) + statement("return (vec)quad_shuffle_rotate_down((vec)value, delta);"); + else + statement("return (vec)simd_shuffle_rotate_down((vec)value, delta);"); + end_scope(); + statement(""); + break; + + // C++ disallows partial specializations of function templates, + // hence the use of a struct. + // clang-format off +#define FUNC_SUBGROUP_CLUSTERED(spv, msl, combine, op, ident) \ + case SPVFuncImplSubgroupClustered##spv: \ + statement("template"); \ + statement("struct spvClustered" #spv "Detail;"); \ + statement(""); \ + statement("// Base cases"); \ + statement("template<>"); \ + statement("struct spvClustered" #spv "Detail<1, 0>"); \ + begin_scope(); \ + statement("template"); \ + statement("static T op(T value, uint)"); \ + begin_scope(); \ + statement("return value;"); \ + end_scope(); \ + end_scope_decl(); \ + statement(""); \ + statement("template"); \ + statement("struct spvClustered" #spv "Detail<1, offset>"); \ + begin_scope(); \ + statement("template"); \ + statement("static T op(T value, uint lid)"); \ + begin_scope(); \ + statement("// If the target lane is inactive, then return identity."); \ + if (msl_options.use_quadgroup_operation()) \ + statement("if (!extract_bits((quad_vote::vote_t)quad_active_threads_mask(), (lid ^ offset), 1))"); \ + else \ + statement("if (!extract_bits(as_type((simd_vote::vote_t)simd_active_threads_mask())[(lid ^ offset) / 32], (lid ^ offset) % 32, 1))"); \ + statement(" return " #ident ";"); \ + if (msl_options.use_quadgroup_operation()) \ + statement("return quad_shuffle_xor(value, offset);"); \ + else \ + statement("return simd_shuffle_xor(value, offset);"); \ + end_scope(); \ + end_scope_decl(); \ + statement(""); \ + statement("template<>"); \ + statement("struct spvClustered" #spv "Detail<4, 0>"); \ + begin_scope(); \ + statement("template"); \ + statement("static T op(T value, uint)"); \ + begin_scope(); \ + statement("return quad_" #msl "(value);"); \ + end_scope(); \ + end_scope_decl(); \ + statement(""); \ + statement("template"); \ + statement("struct spvClustered" #spv "Detail<4, offset>"); \ + begin_scope(); \ + statement("template"); \ + statement("static T op(T value, uint lid)"); \ + begin_scope(); \ + statement("// Here, we care if any of the lanes in the quad are active."); \ + statement("uint quad_mask = extract_bits(as_type((simd_vote::vote_t)simd_active_threads_mask())[(lid ^ offset) / 32], ((lid ^ offset) % 32) & ~3, 4);"); \ + statement("if (!quad_mask)"); \ + statement(" return " #ident ";"); \ + statement("// But we need to make sure we shuffle from an active lane."); \ + if (msl_options.use_quadgroup_operation()) \ + SPIRV_CROSS_THROW("Subgroup size with quadgroup operation cannot exceed 4."); \ + else \ + statement("return simd_shuffle(quad_" #msl "(value), ((lid ^ offset) & ~3) | ctz(quad_mask));"); \ + end_scope(); \ + end_scope_decl(); \ + statement(""); \ + statement("// General case"); \ + statement("template"); \ + statement("struct spvClustered" #spv "Detail"); \ + begin_scope(); \ + statement("template"); \ + statement("static T op(T value, uint lid)"); \ + begin_scope(); \ + statement("return " combine(msl, op, "spvClustered" #spv "Detail::op(value, lid)", "spvClustered" #spv "Detail::op(value, lid)") ";"); \ + end_scope(); \ + end_scope_decl(); \ + statement(""); \ + statement("template"); \ + statement("T spvClustered_" #msl "(T value, uint lid)"); \ + begin_scope(); \ + statement("return spvClustered" #spv "Detail::op(value, lid);"); \ + end_scope(); \ + statement(""); \ + break +#define BINOP(msl, op, l, r) l " " #op " " r +#define BINFUNC(msl, op, l, r) #msl "(" l ", " r ")" + + FUNC_SUBGROUP_CLUSTERED(Add, sum, BINOP, +, 0); + FUNC_SUBGROUP_CLUSTERED(Mul, product, BINOP, *, 1); + FUNC_SUBGROUP_CLUSTERED(Min, min, BINFUNC, , numeric_limits::max()); + FUNC_SUBGROUP_CLUSTERED(Max, max, BINFUNC, , numeric_limits::min()); + FUNC_SUBGROUP_CLUSTERED(And, and, BINOP, &, ~T(0)); + FUNC_SUBGROUP_CLUSTERED(Or, or, BINOP, |, 0); + FUNC_SUBGROUP_CLUSTERED(Xor, xor, BINOP, ^, 0); + // clang-format on +#undef FUNC_SUBGROUP_CLUSTERED +#undef BINOP +#undef BINFUNC + case SPVFuncImplQuadBroadcast: statement("template"); statement("inline T spvQuadBroadcast(T value, uint lane)"); @@ -6655,8 +7448,8 @@ void CompilerMSL::emit_custom_functions() "samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); - statement("ycbcr.br = plane1.sample(samp, coord, spvForward(options)...).rg;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); + statement("ycbcr.br = plane1.sample(samp, coord, options...).rg;"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6668,9 +7461,9 @@ void CompilerMSL::emit_custom_functions() "texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); - statement("ycbcr.b = plane1.sample(samp, coord, spvForward(options)...).r;"); - statement("ycbcr.r = plane2.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); + statement("ycbcr.b = plane1.sample(samp, coord, options...).r;"); + statement("ycbcr.r = plane2.sample(samp, coord, options...).r;"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6682,15 +7475,15 @@ void CompilerMSL::emit_custom_functions() "plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("if (fract(coord.x * plane1.get_width()) != 0.0)"); begin_scope(); - statement("ycbcr.br = vec(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), 0.5).rg);"); + statement("ycbcr.br = vec(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), 0.5).rg);"); end_scope(); statement("else"); begin_scope(); - statement("ycbcr.br = plane1.sample(samp, coord, spvForward(options)...).rg;"); + statement("ycbcr.br = plane1.sample(samp, coord, options...).rg;"); end_scope(); statement("return ycbcr;"); end_scope(); @@ -6703,18 +7496,18 @@ void CompilerMSL::emit_custom_functions() "plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("if (fract(coord.x * plane1.get_width()) != 0.0)"); begin_scope(); - statement("ycbcr.b = T(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), 0.5).r);"); - statement("ycbcr.r = T(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 0)), 0.5).r);"); + statement("ycbcr.b = T(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), 0.5).r);"); + statement("ycbcr.r = T(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., int2(1, 0)), 0.5).r);"); end_scope(); statement("else"); begin_scope(); - statement("ycbcr.b = plane1.sample(samp, coord, spvForward(options)...).r;"); - statement("ycbcr.r = plane2.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.b = plane1.sample(samp, coord, options...).r;"); + statement("ycbcr.r = plane2.sample(samp, coord, options...).r;"); end_scope(); statement("return ycbcr;"); end_scope(); @@ -6727,10 +7520,10 @@ void CompilerMSL::emit_custom_functions() "plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("int2 offs = int2(fract(coord.x * plane1.get_width()) != 0.0 ? 1 : -1, 0);"); - statement("ycbcr.br = vec(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., offs), 0.25).rg);"); + statement("ycbcr.br = vec(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., offs), 0.25).rg);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6742,12 +7535,12 @@ void CompilerMSL::emit_custom_functions() "plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("int2 offs = int2(fract(coord.x * plane1.get_width()) != 0.0 ? 1 : -1, 0);"); - statement("ycbcr.b = T(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., offs), 0.25).r);"); - statement("ycbcr.r = T(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., offs), 0.25).r);"); + statement("ycbcr.b = T(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., offs), 0.25).r);"); + statement("ycbcr.r = T(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., offs), 0.25).r);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6759,12 +7552,12 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract(round(coord * float2(plane0.get_width(), plane0.get_height())) * 0.5);"); - statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).rg);"); + statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).rg);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6776,16 +7569,16 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract(round(coord * float2(plane0.get_width(), plane0.get_height())) * 0.5);"); - statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); - statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane2.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane2.sample(samp, coord, options..., int2(0, 1)), " + "plane2.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6797,13 +7590,13 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0.5, " "0)) * 0.5);"); - statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).rg);"); + statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).rg);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6815,17 +7608,17 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0.5, " "0)) * 0.5);"); - statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); - statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane2.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane2.sample(samp, coord, options..., int2(0, 1)), " + "plane2.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6837,13 +7630,13 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0, " "0.5)) * 0.5);"); - statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).rg);"); + statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).rg);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6855,17 +7648,17 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0, " "0.5)) * 0.5);"); - statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); - statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane2.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane2.sample(samp, coord, options..., int2(0, 1)), " + "plane2.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6877,13 +7670,13 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0.5, " "0.5)) * 0.5);"); - statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).rg);"); + statement("ycbcr.br = vec(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).rg);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -6895,17 +7688,17 @@ void CompilerMSL::emit_custom_functions() "texture2d plane1, texture2d plane2, sampler samp, float2 coord, LodOptions... options)"); begin_scope(); statement("vec ycbcr = vec(0, 0, 0, 1);"); - statement("ycbcr.g = plane0.sample(samp, coord, spvForward(options)...).r;"); + statement("ycbcr.g = plane0.sample(samp, coord, options...).r;"); statement("float2 ab = fract((round(coord * float2(plane0.get_width(), plane0.get_height())) - float2(0.5, " "0.5)) * 0.5);"); - statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, spvForward(options)...), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane1.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane1.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); - statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, spvForward(options)...), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 0)), ab.x), " - "mix(plane2.sample(samp, coord, spvForward(options)..., int2(0, 1)), " - "plane2.sample(samp, coord, spvForward(options)..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.b = T(mix(mix(plane1.sample(samp, coord, options...), " + "plane1.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane1.sample(samp, coord, options..., int2(0, 1)), " + "plane1.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); + statement("ycbcr.r = T(mix(mix(plane2.sample(samp, coord, options...), " + "plane2.sample(samp, coord, options..., int2(1, 0)), ab.x), " + "mix(plane2.sample(samp, coord, options..., int2(0, 1)), " + "plane2.sample(samp, coord, options..., int2(1, 1)), ab.x), ab.y).r);"); statement("return ycbcr;"); end_scope(); statement(""); @@ -7187,10 +7980,8 @@ void CompilerMSL::emit_custom_functions() statement(" ycbcr_samp.get_chroma_filter() == spvChromaFilter::nearest)"); begin_scope(); statement("if (!is_null_texture(plane2))"); - statement(" return spvChromaReconstructNearest(plane0, plane1, plane2, samp, coord,"); - statement(" spvForward(options)...);"); - statement( - "return spvChromaReconstructNearest(plane0, plane1, samp, coord, spvForward(options)...);"); + statement(" return spvChromaReconstructNearest(plane0, plane1, plane2, samp, coord, options...);"); + statement("return spvChromaReconstructNearest(plane0, plane1, samp, coord, options...);"); end_scope(); // if (resolution == 422 || chroma_filter == nearest) statement("switch (ycbcr_samp.get_resolution())"); begin_scope(); @@ -7203,18 +7994,18 @@ void CompilerMSL::emit_custom_functions() statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear422CositedEven("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear422CositedEven("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); statement("case spvXChromaLocation::midpoint:"); statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear422Midpoint("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear422Midpoint("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); end_scope(); // switch (x_chroma_offset) end_scope(); // case 422: statement("case spvFormatResolution::_420:"); @@ -7229,18 +8020,18 @@ void CompilerMSL::emit_custom_functions() statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear420XCositedEvenYCositedEven("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear420XCositedEvenYCositedEven("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); statement("case spvYChromaLocation::midpoint:"); statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear420XCositedEvenYMidpoint("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear420XCositedEvenYMidpoint("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); end_scope(); // switch (y_chroma_offset) end_scope(); // case x::cosited_even: statement("case spvXChromaLocation::midpoint:"); @@ -7251,31 +8042,30 @@ void CompilerMSL::emit_custom_functions() statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear420XMidpointYCositedEven("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear420XMidpointYCositedEven("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); statement("case spvYChromaLocation::midpoint:"); statement(" if (!is_null_texture(plane2))"); statement(" return spvChromaReconstructLinear420XMidpointYMidpoint("); statement(" plane0, plane1, plane2, samp,"); - statement(" coord, spvForward(options)...);"); + statement(" coord, options...);"); statement(" return spvChromaReconstructLinear420XMidpointYMidpoint("); statement(" plane0, plane1, samp, coord,"); - statement(" spvForward(options)...);"); + statement(" options...);"); end_scope(); // switch (y_chroma_offset) end_scope(); // case x::midpoint end_scope(); // switch (x_chroma_offset) end_scope(); // case 420: end_scope(); // switch (resolution) end_scope(); // if (multiplanar) - statement("return plane0.sample(samp, coord, spvForward(options)...);"); + statement("return plane0.sample(samp, coord, options...);"); end_scope(); // do_sample() statement("template "); statement("vec sample(float2 coord, LodOptions... options) const thread"); begin_scope(); - statement( - "vec s = spvTextureSwizzle(do_sample(coord, spvForward(options)...), swizzle);"); + statement("vec s = spvTextureSwizzle(do_sample(coord, options...), swizzle);"); statement("if (ycbcr_samp.get_ycbcr_model() == spvYCbCrModelConversion::rgb_identity)"); statement(" return s;"); statement(""); @@ -7387,36 +8177,37 @@ void CompilerMSL::emit_custom_functions() case SPVFuncImplVariableSizedDescriptor: statement("template"); - statement("struct spvBufferDescriptor"); + statement("struct spvBufferDescriptor;"); + statement(""); + statement("template"); + statement("struct spvBufferDescriptor"); begin_scope(); - statement("T value;"); + statement("device T* value;"); statement("int length;"); - statement("const device T& operator -> () const device"); - begin_scope(); - statement("return value;"); - end_scope(); - statement("const device T& operator * () const device"); - begin_scope(); - statement("return value;"); - end_scope(); + statement("int padding;"); end_scope_decl(); statement(""); break; case SPVFuncImplVariableDescriptorArray: - statement("template"); - statement("struct spvDescriptorArray"); - begin_scope(); - statement("spvDescriptorArray(const device spvDescriptor* ptr) : ptr(ptr)"); - begin_scope(); - end_scope(); - statement("const device T& operator [] (size_t i) const"); - begin_scope(); - statement("return ptr[i].value;"); - end_scope(); - statement("const device spvDescriptor* ptr;"); - end_scope_decl(); - statement(""); + if (spv_function_implementations.count(SPVFuncImplVariableDescriptor) != 0) + { + statement("template"); + statement("struct spvDescriptorArray"); + begin_scope(); + statement("spvDescriptorArray(const device spvDescriptor* ptr_) : ptr(&ptr_->value) {}"); + statement("spvDescriptorArray(const device void *ptr_) : spvDescriptorArray(static_cast*>(ptr_)) {}"); + statement("const device T& operator [] (size_t i) const { return ptr[i]; }"); + statement("const device T* ptr;"); + end_scope_decl(); + statement(""); + } + else + { + statement("template"); + statement("struct spvDescriptorArray;"); + statement(""); + } if (msl_options.runtime_array_rich_descriptor && spv_function_implementations.count(SPVFuncImplVariableSizedDescriptor) != 0) @@ -7424,17 +8215,10 @@ void CompilerMSL::emit_custom_functions() statement("template"); statement("struct spvDescriptorArray"); begin_scope(); - statement("spvDescriptorArray(const device spvBufferDescriptor* ptr) : ptr(ptr)"); - begin_scope(); - end_scope(); - statement("const device T* operator [] (size_t i) const"); - begin_scope(); - statement("return ptr[i].value;"); - end_scope(); - statement("const int length(int i) const"); - begin_scope(); - statement("return ptr[i].length;"); - end_scope(); + statement("spvDescriptorArray(const device spvBufferDescriptor* ptr_) : ptr(ptr_) {}"); + statement("spvDescriptorArray(const device void *ptr_) : spvDescriptorArray(static_cast*>(ptr_)) {}"); + statement("device T* operator [] (size_t i) const { return ptr[i].value; }"); + statement("int length(int i) const { return ptr[i].length; }"); statement("const device spvBufferDescriptor* ptr;"); end_scope_decl(); statement(""); @@ -7450,6 +8234,88 @@ void CompilerMSL::emit_custom_functions() statement(""); break; + case SPVFuncImplPaddedArrayElement: + // .data is used in access chain. + statement("template "); + statement("struct spvPaddedArrayElement { T data; char padding[stride - sizeof(T)]; };"); + statement(""); + break; + + case SPVFuncImplReduceAdd: + // Metal doesn't support __builtin_reduce_add or simd_reduce_add, so we need this. + // Metal also doesn't support the other vector builtins, which would have been useful to make this a single template. + + statement("template "); + statement("T reduce_add(vec v) { return v.x + v.y; }"); + + statement("template "); + statement("T reduce_add(vec v) { return v.x + v.y + v.z; }"); + + statement("template "); + statement("T reduce_add(vec v) { return v.x + v.y + v.z + v.w; }"); + + statement(""); + break; + + case SPVFuncImplImageFence: + statement("template "); + statement("void spvImageFence(ImageT img) { img.fence(); }"); + statement(""); + break; + + case SPVFuncImplTextureCast: + statement("template "); + statement("T spvTextureCast(U img)"); + begin_scope(); + // MSL complains if you try to cast the texture itself, but casting the reference type is ... ok? *shrug* + // Gotta go what you gotta do I suppose. + statement("return reinterpret_cast(img);"); + end_scope(); + statement(""); + break; + + case SPVFuncImplMulExtended: + // Compiler may hit an internal error with mulhi, but doesn't when encapsulated for some reason. + statement("template"); + statement("[[clang::optnone]] T spvMulExtended(V l, V r)"); + begin_scope(); + statement("return T{U(l * r), U(mulhi(l, r))};"); + end_scope(); + statement(""); + break; + + case SPVFuncImplSetMeshOutputsEXT: + statement("void spvSetMeshOutputsEXT(uint gl_LocalInvocationIndex, threadgroup uint2& spvMeshSizes, uint vertexCount, uint primitiveCount)"); + begin_scope(); + statement("if (gl_LocalInvocationIndex == 0)"); + begin_scope(); + statement("spvMeshSizes.x = vertexCount;"); + statement("spvMeshSizes.y = primitiveCount;"); + end_scope(); + end_scope(); + statement(""); + break; + + case SPVFuncImplAssume: + statement_no_indent("#if defined(__has_builtin)"); + statement_no_indent("#if !defined(SPV_ASSUME) && __has_builtin(__builtin_assume)"); + statement_no_indent("#define SPV_ASSUME(x) __builtin_assume(x);"); + statement_no_indent("#endif"); + statement_no_indent("#if !defined(SPV_EXPECT) && __has_builtin(__builtin_expect)"); + statement_no_indent("#define SPV_EXPECT(x, y) __builtin_expect(x, y);"); + statement_no_indent("#endif"); + statement_no_indent("#endif"); + + statement_no_indent("#ifndef SPV_ASSUME"); + statement_no_indent("#define SPV_ASSUME(x)"); + statement_no_indent("#endif"); + + statement_no_indent("#ifndef SPV_EXPECT"); + statement_no_indent("#define SPV_EXPECT(x, y) x"); + statement_no_indent("#endif"); + + break; + default: break; } @@ -7551,6 +8417,24 @@ void CompilerMSL::emit_resources() emit_interface_block(patch_stage_out_var_id); emit_interface_block(stage_in_var_id); emit_interface_block(patch_stage_in_var_id); + + if (get_execution_model() == ExecutionModelMeshEXT) + { + auto &execution = get_entry_point(); + const char *topology = ""; + if (execution.flags.get(ExecutionModeOutputTrianglesEXT)) + topology = "topology::triangle"; + else if (execution.flags.get(ExecutionModeOutputLinesEXT)) + topology = "topology::line"; + else if (execution.flags.get(ExecutionModeOutputPoints)) + topology = "topology::point"; + + const char *per_vertex = mesh_out_per_vertex ? "spvPerVertex" : "float4"; + const char *per_primitive = mesh_out_per_primitive ? "spvPerPrimitive" : "void"; + statement("using spvMesh_t = mesh<", per_vertex, ", ", per_primitive, ", ", execution.output_vertices, ", ", + execution.output_primitives, ", ", topology, ">;"); + statement(""); + } } // Emit declarations for the specialization Metal function constants @@ -7558,8 +8442,16 @@ void CompilerMSL::emit_specialization_constants_and_structs() { SpecializationConstant wg_x, wg_y, wg_z; ID workgroup_size_id = get_work_group_size_specialization_constants(wg_x, wg_y, wg_z); - bool emitted = false; + if (workgroup_size_id == 0 && is_mesh_shader()) + { + auto &execution = get_entry_point(); + statement("constant uint3 ", builtin_to_glsl(BuiltInWorkgroupSize, StorageClassWorkgroup), + " [[maybe_unused]] = ", "uint3(", execution.workgroup_size.x, ", ", execution.workgroup_size.y, ", ", + execution.workgroup_size.z, ");"); + statement(""); + } + bool emitted = false; unordered_set declared_structs; unordered_set aligned_structs; @@ -7574,7 +8466,7 @@ void CompilerMSL::emit_specialization_constants_and_structs() mark_scalar_layout_structs(type); }); - bool builtin_block_type_is_required = false; + bool builtin_block_type_is_required = is_mesh_shader(); // Very special case. If gl_PerVertex is initialized as an array (tessellation) // we have to potentially emit the gl_PerVertex struct type so that we can emit a constant LUT. ir.for_each_typed_id([&](uint32_t, SPIRConstant &c) { @@ -7657,21 +8549,38 @@ void CompilerMSL::emit_specialization_constants_and_structs() if (unique_func_constants[constant_id] == c.self) statement("constant ", sc_type_name, " ", sc_tmp_name, " [[function_constant(", constant_id, ")]];"); - statement("constant ", sc_type_name, " ", sc_name, " = is_function_constant_defined(", sc_tmp_name, - ") ? ", bitcast_expression(type, sc_tmp_type, sc_tmp_name), " : ", constant_expression(c), - ";"); + // RenderDoc and other instrumentation may reuse the same SpecId with different base types. + // We deduplicate to one [[function_constant(id)]] temp and then initialize all variants from it. + // Metal forbids as_type to/from 'bool', so if either side is Boolean, avoid bitcasting here and + // prefer a value cast via a constructor instead (e.g. uint(tmp) / float(tmp) / bool(tmp)). + // This preserves expected toggle semantics and prevents illegal MSL like as_type(bool_tmp). + { + string sc_true_expr; + if (sc_tmp_type == type.basetype) + sc_true_expr = sc_tmp_name; + else if (sc_tmp_type == SPIRType::Boolean || type.basetype == SPIRType::Boolean) + sc_true_expr = join(sc_type_name, "(", sc_tmp_name, ")"); + else + sc_true_expr = bitcast_expression(type, sc_tmp_type, sc_tmp_name); + statement("constant ", sc_type_name, " ", sc_name, " = is_function_constant_defined(", sc_tmp_name, + ") ? ", sc_true_expr, " : ", constant_expression(c), ";"); + } } else if (has_decoration(c.self, DecorationSpecId)) { // Fallback to macro overrides. + uint32_t constant_id = get_decoration(c.self, DecorationSpecId); c.specialization_constant_macro_name = - constant_value_macro_name(get_decoration(c.self, DecorationSpecId)); + constant_value_macro_name(constant_id); statement("#ifndef ", c.specialization_constant_macro_name); statement("#define ", c.specialization_constant_macro_name, " ", constant_expression(c)); statement("#endif"); statement("constant ", sc_type_name, " ", sc_name, " = ", c.specialization_constant_macro_name, ";"); + + // Record the usage of macro + constant_macro_ids.insert(constant_id); } else { @@ -7837,7 +8746,7 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id const uint32_t indices[2] = { i, interface_index }; AccessChainMeta meta; expr += access_chain_internal(stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta, nullptr); if (i + 1 < num_control_points) expr += ", "; } @@ -7873,7 +8782,8 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal(stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, + &meta, nullptr); if (!is_matrix(sub_type) && sub_type.basetype != SPIRType::Struct && expr_type.vecsize > sub_type.vecsize) expr += vector_swizzle(sub_type.vecsize, 0); @@ -7931,7 +8841,8 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal( stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, + &meta, nullptr); } else expr += to_expression(ptr) + "." + to_member_name(iface_type, interface_index); @@ -7955,7 +8866,8 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal( stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, + &meta, nullptr); } else expr += to_expression(ptr) + "." + to_member_name(iface_type, interface_index); @@ -7975,7 +8887,7 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal(stage_in_ptr_var_id, indices, 2, ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, - &meta); + &meta, nullptr); } else expr += to_expression(ptr) + "." + to_member_name(iface_type, interface_index); @@ -8020,7 +8932,8 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal(stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, + &meta, nullptr); if (expr_type.vecsize > result_type.vecsize) expr += vector_swizzle(result_type.vecsize, 0); if (j + 1 < result_type.columns) @@ -8065,7 +8978,8 @@ bool CompilerMSL::emit_tessellation_io_load(uint32_t result_type_id, uint32_t id AccessChainMeta meta; expr += access_chain_internal(stage_in_ptr_var_id, indices, 2, - ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, &meta); + ACCESS_CHAIN_INDEX_IS_LITERAL_BIT | ACCESS_CHAIN_PTR_CHAIN_BIT, + &meta, nullptr); if (expr_type.vecsize > result_type.vecsize) expr += vector_swizzle(result_type.vecsize, 0); @@ -8110,7 +9024,6 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l auto *var = maybe_get_backing_variable(ops[2]); bool patch = false; bool flat_data = false; - bool ptr_is_chain = false; bool flatten_composites = false; bool is_block = false; @@ -8131,12 +9044,6 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l // Patch inputs are treated as normal block IO variables, so they don't deal with this path at all. if (patch && (!is_block || is_arrayed || var->storage == StorageClassInput)) flat_data = false; - - // We might have a chained access chain, where - // we first take the access chain to the control point, and then we chain into a member or something similar. - // In this case, we need to skip gl_in/gl_out remapping. - // Also, skip ptr chain for patches. - ptr_is_chain = var->self != ID(ops[2]); } bool builtin_variable = false; @@ -8157,10 +9064,27 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l if (variable_is_flat) { + if (auto *ptr_expr = maybe_get(ops[2])) + { + // Too many edge cases in incrementally resolving tessellation access chains. + // Only reasonable option is to completely rematerialize the chain from the start. + SmallVector rematerialize_ops; + rematerialize_ops.push_back(ops[0]); + rematerialize_ops.push_back(ops[1]); + + for (auto expr : ptr_expr->implied_read_expressions) + rematerialize_ops.push_back(expr); + + for (uint32_t i = 3; i < length; i++) + rematerialize_ops.push_back(ops[i]); + + return emit_tessellation_access_chain(rematerialize_ops.data(), uint32_t(rematerialize_ops.size())); + } + // If output is masked, it is emitted as a "normal" variable, just go through normal code paths. // Only check this for the first level of access chain. // Dealing with this for partial access chains should be possible, but awkward. - if (var->storage == StorageClassOutput && !ptr_is_chain) + if (var->storage == StorageClassOutput) { bool masked = false; if (is_block) @@ -8187,7 +9111,7 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l indices.reserve(length - 3 + 1); - uint32_t first_non_array_index = (ptr_is_chain ? 3 : 4) - (patch ? 1 : 0); + uint32_t first_non_array_index = 4 - (patch ? 1 : 0); VariableID stage_var_id; if (patch) @@ -8195,8 +9119,9 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l else stage_var_id = var->storage == StorageClassInput ? stage_in_ptr_var_id : stage_out_ptr_var_id; - VariableID ptr = ptr_is_chain ? VariableID(ops[2]) : stage_var_id; - if (!ptr_is_chain && !patch) + VariableID ptr = stage_var_id; + + if (!patch) { // Index into gl_in/gl_out with first array index. indices.push_back(ops[first_non_array_index - 1]); @@ -8207,17 +9132,7 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l uint32_t const_mbr_id = next_id++; uint32_t index = get_extended_decoration(ops[2], SPIRVCrossDecorationInterfaceMemberIndex); - // If we have a pointer chain expression, and we are no longer pointing to a composite - // object, we are in the clear. There is no longer a need to flatten anything. - bool further_access_chain_is_trivial = false; - if (ptr_is_chain && flatten_composites) - { - auto &ptr_type = expression_type(ptr); - if (!is_array(ptr_type) && !is_matrix(ptr_type) && ptr_type.basetype != SPIRType::Struct) - further_access_chain_is_trivial = true; - } - - if (!further_access_chain_is_trivial && (flatten_composites || is_block)) + if (flatten_composites || is_block) { uint32_t i = first_non_array_index; auto *type = &get_variable_element_type(*var); @@ -8313,42 +9228,8 @@ bool CompilerMSL::emit_tessellation_access_chain(const uint32_t *ops, uint32_t l // We use the pointer to the base of the input/output array here, // so this is always a pointer chain. - string e; - - if (!ptr_is_chain) - { - // This is the start of an access chain, use ptr_chain to index into control point array. - e = access_chain(ptr, indices.data(), uint32_t(indices.size()), result_ptr_type, &meta, !patch); - } - else - { - // If we're accessing a struct, we need to use member indices which are based on the IO block, - // not actual struct type, so we have to use a split access chain here where - // first path resolves the control point index, i.e. gl_in[index], and second half deals with - // looking up flattened member name. - - // However, it is possible that we partially accessed a struct, - // by taking pointer to member inside the control-point array. - // For this case, we fall back to a natural access chain since we have already dealt with remapping struct members. - // One way to check this here is if we have 2 implied read expressions. - // First one is the gl_in/gl_out struct itself, then an index into that array. - // If we have traversed further, we use a normal access chain formulation. - auto *ptr_expr = maybe_get(ptr); - bool split_access_chain_formulation = flatten_composites && ptr_expr && - ptr_expr->implied_read_expressions.size() == 2 && - !further_access_chain_is_trivial; - - if (split_access_chain_formulation) - { - e = join(to_expression(ptr), - access_chain_internal(stage_var_id, indices.data(), uint32_t(indices.size()), - ACCESS_CHAIN_CHAIN_ONLY_BIT, &meta)); - } - else - { - e = access_chain_internal(ptr, indices.data(), uint32_t(indices.size()), 0, &meta); - } - } + // This is the start of an access chain, use ptr_chain to index into control point array. + auto e = access_chain(ptr, indices.data(), uint32_t(indices.size()), result_ptr_type, &meta, !patch); // Get the actual type of the object that was accessed. If it's a vector type and we changed it, // then we'll need to add a swizzle. @@ -8455,13 +9336,18 @@ bool CompilerMSL::is_out_of_bounds_tessellation_level(uint32_t id_lhs) } bool CompilerMSL::prepare_access_chain_for_scalar_access(std::string &expr, const SPIRType &type, - spv::StorageClass storage, bool &is_packed) + StorageClass storage, bool &is_packed) { // If there is any risk of writes happening with the access chain in question, // and there is a risk of concurrent write access to other components, // we must cast the access chain to a plain pointer to ensure we only access the exact scalars we expect. // The MSL compiler refuses to allow component-level access for any non-packed vector types. - if (!is_packed && (storage == StorageClassStorageBuffer || storage == StorageClassWorkgroup)) + // MSL refuses to take address or reference to vector component, even for packed types, so just force + // through the pointer cast. No much we can do sadly. + // For packed types, we could technically omit this if we know the reference does not have to turn into a pointer + // of some kind, but that requires external analysis passes to figure out, and + // this case is likely rare enough that we don't need to bother. + if (storage == StorageClassStorageBuffer || storage == StorageClassWorkgroup) { const char *addr_space = storage == StorageClassWorkgroup ? "threadgroup" : "device"; expr = join("((", addr_space, " ", type_to_glsl(type), "*)&", enclose_expression(expr), ")"); @@ -8546,7 +9432,7 @@ void CompilerMSL::fix_up_interpolant_access_chain(const uint32_t *ops, uint32_t // If the physical type of a physical buffer pointer has been changed // to a ulong or ulongn vector, add a cast back to the pointer type. -void CompilerMSL::check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type) +bool CompilerMSL::check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type) { auto *p_physical_type = maybe_get(physical_type); if (p_physical_type && @@ -8557,7 +9443,10 @@ void CompilerMSL::check_physical_type_cast(std::string &expr, const SPIRType *ty expr += ".x"; expr = join("((", type_to_glsl(*type), ")", expr, ")"); + return true; } + + return false; } // Override for MSL-specific syntax instructions @@ -8600,10 +9489,31 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) } else { - // Sample mask input for Metal is not an array - if (BuiltIn(get_decoration(ptr, DecorationBuiltIn)) == BuiltInSampleMask) - set_decoration(id, DecorationBuiltIn, BuiltInSampleMask); - CompilerGLSL::emit_instruction(instruction); + auto is_sample_mask = BuiltIn(get_decoration(ptr, DecorationBuiltIn)) == BuiltInSampleMask; + auto ptr_storage = get_expression_effective_storage_class(ptr); + auto *ptr_var = maybe_get_backing_variable(ptr); + + // More edge cases ... Normally composite outputs are lowered at the end, + // but that's not the case for clip-cull arrays. + if (ptr_var && ptr_storage == StorageClassOutput && is_builtin_variable(*ptr_var) && + !is_sample_mask && is_array(get(ops[0]))) + { + emit_uninitialized_temporary_expression(ops[0], id); + auto &type = get(ops[0]); + if (type.array.size() != 1) + SPIRV_CROSS_THROW("Cannot load array of clip-cull distances from array of array."); + if (!type.array_size_literal.front()) + SPIRV_CROSS_THROW("Cannot load array of clip-cull distances from spec constant array size."); + for (uint32_t i = 0; i < type.array[0]; i++) + statement(to_expression(id), "[", i, "] = ", to_expression(ptr), "[", i, "];"); + } + else + { + // Sample mask input for Metal is not an array + if (is_sample_mask) + set_decoration(id, DecorationBuiltIn, BuiltInSampleMask); + CompilerGLSL::emit_instruction(instruction); + } } break; } @@ -8773,31 +9683,39 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) break; } + case OpSMod: + MSL_BFOP(spvSMod); + break; + case OpFRem: MSL_BFOP(fmod); break; case OpFMul: - if (msl_options.invariant_float_math || has_decoration(ops[1], DecorationNoContraction)) + if (msl_options.invariant_float_math || has_legacy_nocontract(ops[0], ops[1])) MSL_BFOP(spvFMul); else MSL_BOP(*); break; case OpFAdd: - if (msl_options.invariant_float_math || has_decoration(ops[1], DecorationNoContraction)) + if (msl_options.invariant_float_math || has_legacy_nocontract(ops[0], ops[1])) MSL_BFOP(spvFAdd); else MSL_BOP(+); break; case OpFSub: - if (msl_options.invariant_float_math || has_decoration(ops[1], DecorationNoContraction)) + if (msl_options.invariant_float_math || has_legacy_nocontract(ops[0], ops[1])) MSL_BFOP(spvFSub); else MSL_BOP(-); break; + case OpFmaKHR: + MSL_TFOP(fma); + break; + // Atomics case OpAtomicExchange: { @@ -8926,7 +9844,12 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) // Metal requires explicit fences to break up RAW hazards, even within the same shader invocation if (msl_options.readwrite_texture_fences && p_var && !has_decoration(p_var->self, DecorationNonWritable)) - statement(to_expression(img_id), ".fence();"); + { + add_spv_func_and_recompile(SPVFuncImplImageFence); + // Need to wrap this with a value type, + // since the Metal headers are broken and do not consider case when the image is a reference. + statement("spvImageFence(", to_expression(img_id), ");"); + } emit_texture_op(instruction, false); break; @@ -8951,6 +9874,7 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) auto &e = set(id, join(to_expression(ops[2]), "_atomic[", coord, "]"), result_type, true); e.loaded_from = var ? var->self : ID(0); + e.access_chain = true; // This is kinda an access chain and should be treated as a dereferenced expression. inherit_expression_dependencies(id, ops[3]); } else @@ -9102,18 +10026,40 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) uint32_t coord_id = ops[3]; emit_uninitialized_temporary_expression(result_type, id); + std::string coord_expr = to_expression(coord_id); auto sampler_expr = to_sampler_expression(image_id); auto *combined = maybe_get(image_id); auto image_expr = combined ? to_expression(combined->image) : to_expression(image_id); + const SPIRType &image_type = expression_type(image_id); + const SPIRType &coord_type = expression_type(coord_id); + + switch (image_type.image.dim) + { + case Dim1D: + if (!msl_options.texture_1D_as_2D) + SPIRV_CROSS_THROW("ImageQueryLod is not supported on 1D textures."); + [[fallthrough]]; + case Dim2D: + if (coord_type.vecsize > 2) + coord_expr = enclose_expression(coord_expr) + ".xy"; + break; + case DimCube: + case Dim3D: + if (coord_type.vecsize > 3) + coord_expr = enclose_expression(coord_expr) + ".xyz"; + break; + default: + SPIRV_CROSS_THROW("Bad image type given to OpImageQueryLod"); + } // TODO: It is unclear if calculcate_clamped_lod also conditionally rounds // the reported LOD based on the sampler. NEAREST miplevel should // round the LOD, but LINEAR miplevel should not round. // Let's hope this does not become an issue ... statement(to_expression(id), ".x = ", image_expr, ".calculate_clamped_lod(", sampler_expr, ", ", - to_expression(coord_id), ");"); + coord_expr, ");"); statement(to_expression(id), ".y = ", image_expr, ".calculate_unclamped_lod(", sampler_expr, ", ", - to_expression(coord_id), ");"); + coord_expr, ");"); register_control_dependent_expression(id); break; } @@ -9216,6 +10162,9 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) if (needs_frag_discard_checks() && (type.storage == StorageClassStorageBuffer || type.storage == StorageClassUniform)) end_scope(); + if (has_decoration(ops[0], DecorationBuiltIn) && get_decoration(ops[0], DecorationBuiltIn) == BuiltInPointSize) + writes_to_point_size = true; + break; } @@ -9226,9 +10175,9 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) case OpControlBarrier: // In GLSL a memory barrier is often followed by a control barrier. - // But in MSL, memory barriers are also control barriers, so don't + // But in MSL, memory barriers are also control barriers (before MSL 3.2), so don't // emit a simple control barrier if a memory barrier has just been emitted. - if (previous_instruction_opcode != OpMemoryBarrier) + if (previous_instruction_opcode != OpMemoryBarrier || msl_options.supports_msl_version(3, 2)) emit_barrier(ops[0], ops[1], ops[2]); break; @@ -9260,7 +10209,7 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) case OpVectorTimesMatrix: case OpMatrixTimesVector: { - if (!msl_options.invariant_float_math && !has_decoration(ops[1], DecorationNoContraction)) + if (!msl_options.invariant_float_math && !has_legacy_nocontract(ops[0], ops[1])) { CompilerGLSL::emit_instruction(instruction); break; @@ -9302,7 +10251,7 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) case OpMatrixTimesMatrix: { - if (!msl_options.invariant_float_math && !has_decoration(ops[1], DecorationNoContraction)) + if (!msl_options.invariant_float_math && !has_legacy_nocontract(ops[0], ops[1])) { CompilerGLSL::emit_instruction(instruction); break; @@ -9323,8 +10272,7 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) enclose_expression(to_unpacked_row_major_matrix_expression(ops[2])), ")"); bool forward = should_forward(ops[2]) && should_forward(ops[3]); - auto &e = emit_op(ops[0], ops[1], expr, forward); - e.need_transpose = true; + emit_transposed_op(ops[0], ops[1], expr, forward); a->need_transpose = true; b->need_transpose = true; inherit_expression_dependencies(ops[1], ops[2]); @@ -9374,33 +10322,13 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) uint32_t op0 = ops[2]; uint32_t op1 = ops[3]; auto &type = get(result_type); + auto &op_type = get(type.member_types[0]); auto input_type = opcode == OpSMulExtended ? int_type : uint_type; - auto &output_type = get_type(result_type); string cast_op0, cast_op1; - auto expected_type = binary_op_bitcast_helper(cast_op0, cast_op1, input_type, op0, op1, false); - - emit_uninitialized_temporary_expression(result_type, result_id); - - string mullo_expr, mulhi_expr; - mullo_expr = join(cast_op0, " * ", cast_op1); - mulhi_expr = join("mulhi(", cast_op0, ", ", cast_op1, ")"); - - auto &low_type = get_type(output_type.member_types[0]); - auto &high_type = get_type(output_type.member_types[1]); - if (low_type.basetype != input_type) - { - expected_type.basetype = input_type; - mullo_expr = join(bitcast_glsl_op(low_type, expected_type), "(", mullo_expr, ")"); - } - if (high_type.basetype != input_type) - { - expected_type.basetype = input_type; - mulhi_expr = join(bitcast_glsl_op(high_type, expected_type), "(", mulhi_expr, ")"); - } - - statement(to_expression(result_id), ".", to_member_name(type, 0), " = ", mullo_expr, ";"); - statement(to_expression(result_id), ".", to_member_name(type, 1), " = ", mulhi_expr, ";"); + binary_op_bitcast_helper(cast_op0, cast_op1, input_type, op0, op1, false); + auto expr = join("spvMulExtended<", type_to_glsl(type), ", ", type_to_glsl(op_type), ">(", cast_op0, ", ", cast_op1, ")"); + emit_op(result_type, result_id, expr, true); break; } @@ -9613,18 +10541,26 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) auto &type = get(ops[0]); auto &input_type = expression_type(ops[2]); - if (opcode != OpBitcast || type.pointer || input_type.pointer) + if (opcode != OpBitcast || is_pointer(type) || is_pointer(input_type)) { string op; + auto input_expr = to_unpacked_expression(ops[2]); - if (type.vecsize == 1 && input_type.vecsize == 1) - op = join("reinterpret_cast<", type_to_glsl(type), ">(", to_unpacked_expression(ops[2]), ")"); + if ((type.vecsize == 1 || is_pointer(type)) && (input_type.vecsize == 1 || is_pointer(input_type))) + op = join("reinterpret_cast<", type_to_glsl(type), ">(", input_expr, ")"); else if (input_type.vecsize == 2) - op = join("reinterpret_cast<", type_to_glsl(type), ">(as_type(", to_unpacked_expression(ops[2]), "))"); + op = join("reinterpret_cast<", type_to_glsl(type), ">(as_type(", input_expr, "))"); else - op = join("as_type<", type_to_glsl(type), ">(reinterpret_cast(", to_unpacked_expression(ops[2]), "))"); + op = join("as_type<", type_to_glsl(type), ">(reinterpret_cast(", input_expr, "))"); - emit_op(ops[0], ops[1], op, should_forward(ops[2])); + auto &expr = emit_op(ops[0], ops[1], op, should_forward(ops[2])); + if (is_pointer(type)) + { + if (auto *backing_var = maybe_get_backing_variable(ops[2])) + expr.loaded_from = backing_var->self; + else + expr.loaded_from = ID(ops[2]); + } inherit_expression_dependencies(ops[1], ops[2]); } else @@ -9633,10 +10569,399 @@ void CompilerMSL::emit_instruction(const Instruction &instruction) break; } + case OpSDot: + case OpUDot: + case OpSUDot: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t vec1 = ops[2]; + uint32_t vec2 = ops[3]; + + auto &input_type1 = expression_type(vec1); + auto &input_type2 = expression_type(vec2); + + string vec1input, vec2input; + auto input_size = input_type1.vecsize; + if (instruction.length == 5) + { + if (ops[4] == PackedVectorFormatPackedVectorFormat4x8Bit) + { + string type = opcode == OpSDot || opcode == OpSUDot ? "char4" : "uchar4"; + vec1input = join("as_type<", type, ">(", to_expression(vec1), ")"); + type = opcode == OpSDot ? "char4" : "uchar4"; + vec2input = join("as_type<", type, ">(", to_expression(vec2), ")"); + input_size = 4; + } + else + SPIRV_CROSS_THROW("Packed vector formats other than 4x8Bit for integer dot product is not supported."); + } + else + { + // Inputs are sign or zero-extended to their target width. + SPIRType::BaseType vec1_expected_type = + opcode != OpUDot ? + to_signed_basetype(input_type1.width) : + to_unsigned_basetype(input_type1.width); + + SPIRType::BaseType vec2_expected_type = + opcode != OpSDot ? + to_unsigned_basetype(input_type2.width) : + to_signed_basetype(input_type2.width); + + vec1input = bitcast_expression(vec1_expected_type, vec1); + vec2input = bitcast_expression(vec2_expected_type, vec2); + } + + auto &type = get(result_type); + + // We'll get the appropriate sign-extend or zero-extend, no matter which type we cast to here. + // The addition in reduce_add is sign-invariant. + auto result_type_cast = join(type_to_glsl(type), input_size); + + string exp = join("reduce_add(", + result_type_cast, "(", vec1input, ") * ", + result_type_cast, "(", vec2input, "))"); + + emit_op(result_type, id, exp, should_forward(vec1) && should_forward(vec2)); + inherit_expression_dependencies(id, vec1); + inherit_expression_dependencies(id, vec2); + break; + } + + case OpSDotAccSat: + case OpUDotAccSat: + case OpSUDotAccSat: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t vec1 = ops[2]; + uint32_t vec2 = ops[3]; + uint32_t acc = ops[4]; + + auto input_type1 = expression_type(vec1); + auto input_type2 = expression_type(vec2); + + string vec1input, vec2input; + if (instruction.length == 6) + { + if (ops[5] == PackedVectorFormatPackedVectorFormat4x8Bit) + { + string type = opcode == OpSDotAccSat || opcode == OpSUDotAccSat ? "char4" : "uchar4"; + vec1input = join("as_type<", type, ">(", to_expression(vec1), ")"); + type = opcode == OpSDotAccSat ? "char4" : "uchar4"; + vec2input = join("as_type<", type, ">(", to_expression(vec2), ")"); + input_type1.vecsize = 4; + input_type2.vecsize = 4; + } + else + SPIRV_CROSS_THROW("Packed vector formats other than 4x8Bit for integer dot product is not supported."); + } + else + { + // Inputs are sign or zero-extended to their target width. + SPIRType::BaseType vec1_expected_type = + opcode != OpUDotAccSat ? + to_signed_basetype(input_type1.width) : + to_unsigned_basetype(input_type1.width); + + SPIRType::BaseType vec2_expected_type = + opcode != OpSDotAccSat ? + to_unsigned_basetype(input_type2.width) : + to_signed_basetype(input_type2.width); + + vec1input = bitcast_expression(vec1_expected_type, vec1); + vec2input = bitcast_expression(vec2_expected_type, vec2); + } + + auto &type = get(result_type); + + SPIRType::BaseType pre_saturate_type = + opcode != OpUDotAccSat ? + to_signed_basetype(type.width) : + to_unsigned_basetype(type.width); + + input_type1.basetype = pre_saturate_type; + input_type2.basetype = pre_saturate_type; + + string exp = join(type_to_glsl(type), "(addsat(reduce_add(", + type_to_glsl(input_type1), "(", vec1input, ") * ", + type_to_glsl(input_type2), "(", vec2input, ")), ", + bitcast_expression(pre_saturate_type, acc), "))"); + + emit_op(result_type, id, exp, should_forward(vec1) && should_forward(vec2)); + inherit_expression_dependencies(id, vec1); + inherit_expression_dependencies(id, vec2); + break; + } + + case OpSetMeshOutputsEXT: + { + flush_variable_declaration(builtin_mesh_primitive_indices_id); + add_spv_func_and_recompile(SPVFuncImplSetMeshOutputsEXT); + statement("spvSetMeshOutputsEXT(gl_LocalInvocationIndex, spvMeshSizes, ", to_unpacked_expression(ops[0]), ", ", to_unpacked_expression(ops[1]), ");"); + break; + } + + case OpAssumeTrueKHR: + { + auto condition = ops[0]; + statement(join("SPV_ASSUME(", to_unpacked_expression(condition), ")")); + break; + } + + case OpExpectKHR: + { + auto result_type = ops[0]; + auto ret = ops[1]; + auto value = ops[2]; + auto exp_value = ops[3]; + + auto exp = join("SPV_EXPECT(", to_unpacked_expression(value), ", ", to_unpacked_expression(exp_value), ")"); + emit_op(result_type, ret, exp, should_forward(value), should_forward(exp_value)); + inherit_expression_dependencies(ret, value); + inherit_expression_dependencies(ret, exp_value); + break; + } + + case OpCooperativeMatrixLoadKHR: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t ptr = ops[2]; + uint32_t layout = ops[3]; + + auto &layout_c = get(layout); + if (layout_c.specialization) + SPIRV_CROSS_THROW("MSL cooperative matrix load does not support spec-constant layout."); + uint32_t layout_val = layout_c.scalar(); + bool col_major = false; + + switch (layout_val) + { + case CooperativeMatrixLayoutRowMajorKHR: + case CooperativeMatrixLayoutColumnMajorKHR: + if (instruction.length < 5) + SPIRV_CROSS_THROW("MSL cooperative matrix load requires Stride for row/column-major layouts."); + col_major = (layout_val == CooperativeMatrixLayoutColumnMajorKHR); + break; + + default: + SPIRV_CROSS_THROW("MSL cooperative matrix load only supports RowMajorKHR and ColumnMajorKHR layouts."); + } + + uint32_t stride = ops[4]; + + emit_uninitialized_temporary_expression(result_type, id); + + auto ptr_expr = to_ptr_expression(ptr); + string stride_expr = to_expression(stride); + + // The pointer operand is allowed to use a different element type than the cooperative matrix component type. + // In that case, cast the pointer and convert the stride from source element units to component element units. + auto &mat_type = get(result_type); + auto &component_type = get(mat_type.parent_type); + auto &ptr_type = expression_type(ptr); + auto &pointee_type = get(ptr_type.parent_type); + if (pointee_type.self != component_type.self) + { + auto addr_space = get_type_address_space(ptr_type, ptr); + ptr_expr = join("reinterpret_cast<", addr_space, " ", type_to_glsl(component_type), "*>(", ptr_expr, ")"); + + uint32_t src_bytes = (pointee_type.width * pointee_type.vecsize) / 8; + uint32_t dst_bytes = (component_type.width * component_type.vecsize) / 8; + if (src_bytes == 0 || dst_bytes == 0) + SPIRV_CROSS_THROW("Cannot determine element size for cooperative matrix load/store."); + + if (src_bytes == dst_bytes) + { + // No conversion needed. + } + else if (src_bytes > dst_bytes && (src_bytes % dst_bytes) == 0) + { + uint32_t multiplier = src_bytes / dst_bytes; + stride_expr = join("(", stride_expr, ") * ", multiplier, "u"); + } + else if (src_bytes < dst_bytes && (dst_bytes % src_bytes) == 0) + { + uint32_t divisor = dst_bytes / src_bytes; + stride_expr = join("(", stride_expr, ") / ", divisor, "u"); + } + else + { + stride_expr = join("((", stride_expr, ") * ", src_bytes, "u) / ", dst_bytes, "u"); + } + } + + if (col_major) + statement("simdgroup_load(", to_expression(id), ", ", + ptr_expr, ", ", stride_expr, ", ulong2(0), true);"); + else + statement("simdgroup_load(", to_expression(id), ", ", + ptr_expr, ", ", stride_expr, ");"); + + register_read(id, ptr, false); + break; + } + + case OpCooperativeMatrixStoreKHR: + { + uint32_t ptr = ops[0]; + uint32_t obj = ops[1]; + uint32_t layout = ops[2]; + + auto &layout_c = get(layout); + if (layout_c.specialization) + SPIRV_CROSS_THROW("MSL cooperative matrix store does not support spec-constant layout."); + uint32_t layout_val = layout_c.scalar(); + bool col_major = false; + + switch (layout_val) + { + case CooperativeMatrixLayoutRowMajorKHR: + case CooperativeMatrixLayoutColumnMajorKHR: + if (instruction.length < 4) + SPIRV_CROSS_THROW("MSL cooperative matrix store requires Stride for row/column-major layouts."); + col_major = (layout_val == CooperativeMatrixLayoutColumnMajorKHR); + break; + + default: + SPIRV_CROSS_THROW("MSL cooperative matrix store only supports RowMajorKHR and ColumnMajorKHR layouts."); + } + + uint32_t stride = ops[3]; + + auto ptr_expr = to_ptr_expression(ptr); + string stride_expr = to_expression(stride); + + // The pointer operand is allowed to use a different element type than the cooperative matrix component type. + // In that case, cast the pointer and convert the stride from source element units to component element units. + auto &mat_type = expression_type(obj); + auto &component_type = get(mat_type.parent_type); + auto &ptr_type = expression_type(ptr); + auto &pointee_type = get(ptr_type.parent_type); + if (pointee_type.self != component_type.self) + { + auto addr_space = get_type_address_space(ptr_type, ptr); + ptr_expr = join("reinterpret_cast<", addr_space, " ", type_to_glsl(component_type), "*>(", ptr_expr, ")"); + + uint32_t src_bytes = (pointee_type.width * pointee_type.vecsize) / 8; + uint32_t dst_bytes = (component_type.width * component_type.vecsize) / 8; + if (src_bytes == 0 || dst_bytes == 0) + SPIRV_CROSS_THROW("Cannot determine element size for cooperative matrix load/store."); + + if (src_bytes == dst_bytes) + { + // No conversion needed. + } + else if (src_bytes > dst_bytes && (src_bytes % dst_bytes) == 0) + { + uint32_t multiplier = src_bytes / dst_bytes; + stride_expr = join("(", stride_expr, ") * ", multiplier, "u"); + } + else if (src_bytes < dst_bytes && (dst_bytes % src_bytes) == 0) + { + uint32_t divisor = dst_bytes / src_bytes; + stride_expr = join("(", stride_expr, ") / ", divisor, "u"); + } + else + { + stride_expr = join("((", stride_expr, ") * ", src_bytes, "u) / ", dst_bytes, "u"); + } + } + + if (col_major) + statement("simdgroup_store(", to_expression(obj), ", ", + ptr_expr, ", ", stride_expr, ", ulong2(0), true);"); + else + statement("simdgroup_store(", to_expression(obj), ", ", + ptr_expr, ", ", stride_expr, ");"); + + register_write(ptr); + break; + } + + case OpCooperativeMatrixMulAddKHR: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + uint32_t A = ops[2], B = ops[3], C = ops[4]; + uint32_t matrix_operands = instruction.length >= 6 ? ops[5] : uint32_t(CooperativeMatrixOperandsMaskNone); + + if (matrix_operands != uint32_t(CooperativeMatrixOperandsMaskNone)) + SPIRV_CROSS_THROW("MSL cooperative matrix muladd does not support setting matrix operands flags."); + + emit_uninitialized_temporary_expression(result_type, id); + statement("simdgroup_multiply_accumulate(", to_expression(id), ", ", + to_unpacked_expression(A), ", ", + to_unpacked_expression(B), ", ", + to_unpacked_expression(C), ");"); + + inherit_expression_dependencies(id, A); + inherit_expression_dependencies(id, B); + inherit_expression_dependencies(id, C); + break; + } + + case OpCooperativeMatrixLengthKHR: + { + uint32_t result_type = ops[0]; + uint32_t id = ops[1]; + auto &coop_type = get(ops[2]); + + if (coop_type.op != OpTypeCooperativeMatrixKHR) + SPIRV_CROSS_THROW("OpCooperativeMatrixLengthKHR requires cooperative matrix type."); + + auto &component_type = get(coop_type.parent_type); + auto coop_type_name = type_to_glsl(coop_type); + auto component_type_name = type_to_glsl(component_type); + + auto expr = join(type_to_glsl(get(result_type)), + "(sizeof(", coop_type_name, "::storage_type) / sizeof(", component_type_name, "))"); + emit_op(result_type, id, expr, true); + break; + } + default: + { + // Prevent GLSL cooperative matrix code from leaking into MSL output. + // Element-wise arithmetic on cooperative matrices is not supported in Metal. + // Should cover any reasonable situation we come across. + if (instruction.length >= 2) + { + bool has_result = false, has_result_type = false; + HasResultAndType(opcode, &has_result, &has_result_type); + + if (has_result_type) + { + auto *type = &get(ops[0]); + while (type && (is_pointer(*type) || is_array(*type))) + type = this->maybe_get(type->parent_type); + if (type->op == OpTypeCooperativeMatrixKHR) + SPIRV_CROSS_THROW("Unsupported operation on cooperative matrix in MSL backend."); + } + + auto is_cooperative_matrix_typed_id = [&](uint32_t id) -> bool { + auto &type = expression_type(id); + return type.op == OpTypeCooperativeMatrixKHR; + }; + + if (opcode == OpCompositeExtract || opcode == OpVectorExtractDynamic) + { + if (instruction.length >= 3 && is_cooperative_matrix_typed_id(ops[2])) + SPIRV_CROSS_THROW("Unsupported extraction from cooperative matrix in MSL backend."); + } + else if (opcode == OpCompositeInsert || opcode == OpVectorInsertDynamic) + { + if ((instruction.length >= 3 && is_cooperative_matrix_typed_id(ops[2])) || + (instruction.length >= 4 && is_cooperative_matrix_typed_id(ops[3]))) + SPIRV_CROSS_THROW("Unsupported operation on cooperative matrix in MSL backend."); + } + } CompilerGLSL::emit_instruction(instruction); break; } + } previous_instruction_opcode = opcode; } @@ -9674,8 +10999,13 @@ void CompilerMSL::emit_texture_op(const Instruction &i, bool sparse) void CompilerMSL::emit_barrier(uint32_t id_exe_scope, uint32_t id_mem_scope, uint32_t id_mem_sem) { - if (get_execution_model() != ExecutionModelGLCompute && !is_tesc_shader()) + auto model = get_execution_model(); + + if (model != ExecutionModelGLCompute && model != ExecutionModelTaskEXT && + model != ExecutionModelMeshEXT && !is_tesc_shader()) + { return; + } uint32_t exe_scope = id_exe_scope ? evaluate_constant_u32(id_exe_scope) : uint32_t(ScopeInvocation); uint32_t mem_scope = id_mem_scope ? evaluate_constant_u32(id_mem_scope) : uint32_t(ScopeInvocation); @@ -9687,10 +11017,20 @@ void CompilerMSL::emit_barrier(uint32_t id_exe_scope, uint32_t id_mem_scope, uin return; string bar_stmt; - if ((msl_options.is_ios() && msl_options.supports_msl_version(1, 2)) || msl_options.supports_msl_version(2)) - bar_stmt = exe_scope < ScopeSubgroup ? "threadgroup_barrier" : "simdgroup_barrier"; + + if (!id_exe_scope && msl_options.supports_msl_version(3, 2)) + { + // Just took 10 years to get a proper barrier, but hey! + bar_stmt = "atomic_thread_fence"; + } else - bar_stmt = "threadgroup_barrier"; + { + if ((msl_options.is_ios() && msl_options.supports_msl_version(1, 2)) || msl_options.supports_msl_version(2)) + bar_stmt = exe_scope < ScopeSubgroup ? "threadgroup_barrier" : "simdgroup_barrier"; + else + bar_stmt = "threadgroup_barrier"; + } + bar_stmt += "("; uint32_t mem_sem = id_mem_sem ? evaluate_constant_u32(id_mem_sem) : uint32_t(MemorySemanticsMaskNone); @@ -9698,7 +11038,8 @@ void CompilerMSL::emit_barrier(uint32_t id_exe_scope, uint32_t id_mem_scope, uin // Use the | operator to combine flags if we can. if (msl_options.supports_msl_version(1, 2)) { - string mem_flags = ""; + string mem_flags; + // For tesc shaders, this also affects objects in the Output storage class. // Since in Metal, these are placed in a device buffer, we have to sync device memory here. if (is_tesc_shader() || @@ -9739,6 +11080,55 @@ void CompilerMSL::emit_barrier(uint32_t id_exe_scope, uint32_t id_mem_scope, uin bar_stmt += "mem_flags::mem_none"; } + if (!id_exe_scope && msl_options.supports_msl_version(3, 2)) + { + // If there's no device-related memory in the barrier, demote to workgroup scope. + // glslang seems to emit device scope even for memoryBarrierShared(). + if (mem_scope == ScopeDevice && + (mem_sem & (MemorySemanticsUniformMemoryMask | + MemorySemanticsImageMemoryMask | + MemorySemanticsCrossWorkgroupMemoryMask)) == 0) + { + mem_scope = ScopeWorkgroup; + } + + // MSL 3.2 only supports seq_cst or relaxed. + if (mem_sem & (MemorySemanticsAcquireReleaseMask | + MemorySemanticsAcquireMask | + MemorySemanticsReleaseMask | + MemorySemanticsSequentiallyConsistentMask)) + { + bar_stmt += ", memory_order_seq_cst"; + } + else + { + bar_stmt += ", memory_order_relaxed"; + } + + switch (mem_scope) + { + case ScopeDevice: + bar_stmt += ", thread_scope_device"; + break; + + case ScopeWorkgroup: + bar_stmt += ", thread_scope_threadgroup"; + break; + + case ScopeSubgroup: + bar_stmt += ", thread_scope_simdgroup"; + break; + + case ScopeInvocation: + bar_stmt += ", thread_scope_thread"; + break; + + default: + // The default argument is device, which is conservative. + break; + } + } + bar_stmt += ");"; statement(bar_stmt); @@ -9788,13 +11178,19 @@ bool CompilerMSL::emit_array_copy(const char *expr, uint32_t lhs_id, uint32_t rh auto *lhs_var = maybe_get_backing_variable(lhs_id); if (lhs_var && lhs_storage == StorageClassStorageBuffer && storage_class_array_is_thread(lhs_var->storage)) lhs_is_array_template = true; - else if (lhs_var && lhs_storage != StorageClassGeneric && type_is_block_like(get(lhs_var->basetype))) + else if (lhs_var && lhs_storage != StorageClassGeneric && type_is_explicit_layout(get(lhs_var->basetype))) lhs_is_array_template = false; auto *rhs_var = maybe_get_backing_variable(rhs_id); if (rhs_var && rhs_storage == StorageClassStorageBuffer && storage_class_array_is_thread(rhs_var->storage)) rhs_is_array_template = true; - else if (rhs_var && rhs_storage != StorageClassGeneric && type_is_block_like(get(rhs_var->basetype))) + else if (rhs_var && rhs_storage != StorageClassGeneric && type_is_explicit_layout(get(rhs_var->basetype))) + rhs_is_array_template = false; + + // Special consideration for clip/culldistance. Normally composites are lowered, but clip/cull is special for reasons ... + if (lhs_var && lhs_storage == StorageClassOutput && is_builtin_variable(*lhs_var)) + lhs_is_array_template = false; + if (rhs_var && rhs_storage == StorageClassOutput && is_builtin_variable(*rhs_var)) rhs_is_array_template = false; // If threadgroup storage qualifiers are *not* used: @@ -9841,15 +11237,7 @@ bool CompilerMSL::emit_array_copy(const char *expr, uint32_t lhs_id, uint32_t rh // we cannot easily detect this case ahead of time since it's // context dependent. We might have to force a recompile here // if this is the only use of array copies in our shader. - if (type.array.size() > 1) - { - if (type.array.size() > kArrayCopyMultidimMax) - SPIRV_CROSS_THROW("Cannot support this many dimensions for arrays of arrays."); - auto func = static_cast(SPVFuncImplArrayCopyMultidimBase + type.array.size()); - add_spv_func_and_recompile(func); - } - else - add_spv_func_and_recompile(SPVFuncImplArrayCopy); + add_spv_func_and_recompile(type.array.size() > 1 ? SPVFuncImplArrayCopyMultidim : SPVFuncImplArrayCopy); const char *tag = nullptr; if (lhs_is_thread_storage && is_constant) @@ -9879,21 +11267,27 @@ bool CompilerMSL::emit_array_copy(const char *expr, uint32_t lhs_id, uint32_t rh else SPIRV_CROSS_THROW("Unknown storage class used for copying arrays."); + // Should be very rare, but mark if we need extra magic template overloads. + if (has_extended_decoration(lhs_id, SPIRVCrossDecorationPhysicalTypeID)) + add_spv_func_and_recompile(SPVFuncImplArrayCopyExtendedDst); + if (has_extended_decoration(rhs_id, SPIRVCrossDecorationPhysicalTypeID)) + add_spv_func_and_recompile(SPVFuncImplArrayCopyExtendedSrc); + // Pass internal array of spvUnsafeArray<> into wrapper functions if (lhs_is_array_template && rhs_is_array_template && !msl_options.force_native_arrays) - statement("spvArrayCopy", tag, type.array.size(), "(", lhs, ".elements, ", to_expression(rhs_id), ".elements);"); + statement("spvArrayCopy", tag, "(", lhs, ".elements, ", to_expression(rhs_id), ".elements);"); if (lhs_is_array_template && !msl_options.force_native_arrays) - statement("spvArrayCopy", tag, type.array.size(), "(", lhs, ".elements, ", to_expression(rhs_id), ");"); + statement("spvArrayCopy", tag, "(", lhs, ".elements, ", to_expression(rhs_id), ");"); else if (rhs_is_array_template && !msl_options.force_native_arrays) - statement("spvArrayCopy", tag, type.array.size(), "(", lhs, ", ", to_expression(rhs_id), ".elements);"); + statement("spvArrayCopy", tag, "(", lhs, ", ", to_expression(rhs_id), ".elements);"); else - statement("spvArrayCopy", tag, type.array.size(), "(", lhs, ", ", to_expression(rhs_id), ");"); + statement("spvArrayCopy", tag, "(", lhs, ", ", to_expression(rhs_id), ");"); } return true; } -uint32_t CompilerMSL::get_physical_tess_level_array_size(spv::BuiltIn builtin) const +uint32_t CompilerMSL::get_physical_tess_level_array_size(BuiltIn builtin) const { if (is_tessellating_triangles()) return builtin == BuiltInTessLevelInner ? 1 : 3; @@ -9964,7 +11358,8 @@ void CompilerMSL::emit_atomic_func_op(uint32_t result_type, uint32_t result_id, { string exp; - auto &type = get_pointee_type(expression_type(obj)); + auto &ptr_type = expression_type(obj); + auto &type = get_pointee_type(ptr_type); auto expected_type = type.basetype; if (opcode == OpAtomicUMax || opcode == OpAtomicUMin) expected_type = to_unsigned_basetype(type.width); @@ -9984,15 +11379,13 @@ void CompilerMSL::emit_atomic_func_op(uint32_t result_type, uint32_t result_id, remapped_type.basetype = expected_type; auto *var = maybe_get_backing_variable(obj); - if (!var) - SPIRV_CROSS_THROW("No backing variable for atomic operation."); - const auto &res_type = get(var->basetype); + const auto *res_type = var ? &get(var->basetype) : nullptr; + assert(type.storage != StorageClassImage || res_type); bool is_atomic_compare_exchange_strong = op1_is_pointer && op1; bool check_discard = opcode != OpAtomicLoad && needs_frag_discard_checks() && - ((res_type.storage == StorageClassUniformConstant && res_type.basetype == SPIRType::Image) || - var->storage == StorageClassStorageBuffer || var->storage == StorageClassUniform); + ptr_type.storage != StorageClassWorkgroup; // Even compare exchange atomics are vec4 on metal for ... reasons :v uint32_t vec4_temporary_id = 0; @@ -10030,25 +11423,97 @@ void CompilerMSL::emit_atomic_func_op(uint32_t result_type, uint32_t result_id, { auto obj_expression = to_expression(obj); auto split_index = obj_expression.find_first_of('@'); + bool needs_reinterpret = opcode == OpAtomicUMax || opcode == OpAtomicUMin || opcode == OpAtomicSMax || opcode == OpAtomicSMin; + needs_reinterpret &= type.basetype != expected_type; + SPIRVariable *backing_var = nullptr; + + // Try to avoid waiting until not force recompile later mode to enable force recompile later + if (needs_reinterpret && (backing_var = maybe_get_backing_variable(obj))) + add_spv_func_and_recompile(SPVFuncImplTextureCast); // Will only be false if we're in "force recompile later" mode. if (split_index != string::npos) - exp += join(obj_expression.substr(0, split_index), ".", op, "(", obj_expression.substr(split_index + 1)); + { + auto coord = obj_expression.substr(split_index + 1); + auto image_expr = obj_expression.substr(0, split_index); + + // Handle problem cases with sign where we need signed min/max on a uint image for example. + // It seems to work to cast the texture type itself, even if it is probably wildly outside of spec, + // but SPIR-V requires this to work. + if (needs_reinterpret && backing_var) + { + assert(spv_function_implementations.count(SPVFuncImplTextureCast) && "Should have been added above"); + + const auto *backing_type = &get(backing_var->basetype); + while (backing_type->op != OpTypeImage) + backing_type = &get(backing_type->parent_type); + + auto img_type = *backing_type; + auto tmp_type = type; + tmp_type.basetype = expected_type; + img_type.image.type = ir.increase_bound_by(1); + set(img_type.image.type, tmp_type); + + image_expr = join("spvTextureCast<", type_to_glsl(img_type, obj), ">(", image_expr, ")"); + } + + exp += join(image_expr, ".", op, "("); + if (ptr_type.storage == StorageClassImage && (res_type->image.arrayed || res_type->image.dim == DimCube)) + { + switch (res_type->image.dim) + { + case Dim1D: + if (msl_options.texture_1D_as_2D) + exp += join("uint2(", coord, ".x, 0), ", coord, ".y"); + else + exp += join(coord, ".x, ", coord, ".y"); + break; + case Dim2D: + exp += join(coord, ".xy, ", coord, ".z"); + break; + case DimCube: + if (!msl_options.supports_msl_version(4, 0)) + SPIRV_CROSS_THROW("Cannot do atomics on Cube textures before 4.0."); + + if (res_type->image.arrayed) + exp += join(coord, ".xy, ", coord, ".z % 6u, ", coord, ".z / 6u"); + else + exp += join(coord, ".xy, ", coord, ".z"); + break; + default: + SPIRV_CROSS_THROW("Cannot do atomics on unknown dimension."); + } + } + else if (ptr_type.storage == StorageClassImage && res_type->image.dim == Dim1D && msl_options.texture_1D_as_2D) + exp += join("uint2(", coord, ", 0)"); + else + exp += coord; + } else + { exp += obj_expression; + } } else { exp += string(op) + "_explicit("; exp += "("; // Emulate texture2D atomic operations - if (res_type.storage == StorageClassUniformConstant && res_type.basetype == SPIRType::Image) + if (ptr_type.storage == StorageClassImage) { + auto &flags = ir.get_decoration_bitset(var->self); + if (decoration_flags_signal_volatile(flags)) + exp += "volatile "; exp += "device"; } + else if (var && ptr_type.storage != StorageClassPhysicalStorageBuffer) + { + exp += get_variable_address_space(*var); + } else { - exp += get_argument_address_space(*var); + // Fallback scenario, could happen for raw pointers. + exp += ptr_type.storage == StorageClassWorkgroup ? "threadgroup" : "device"; } exp += " atomic_"; @@ -10056,9 +11521,7 @@ void CompilerMSL::emit_atomic_func_op(uint32_t result_type, uint32_t result_id, // There is no other way, since C++ does not have explicit signage for atomics. exp += type_to_glsl(remapped_type); exp += "*)"; - - exp += "&"; - exp += to_enclosed_expression(obj); + exp += to_enclosed_pointer_expression(obj); } if (is_atomic_compare_exchange_strong) @@ -10188,22 +11651,107 @@ void CompilerMSL::emit_glsl_op(uint32_t result_type, uint32_t id, uint32_t eop, op = get_remapped_glsl_op(op); + auto &restype = get(result_type); + + // Only precise:: preserves NaN in trancendentals (supposedly, cannot find documentation for this). + const auto drop_nan_inf = FPFastMathModeNotInfMask | FPFastMathModeNotNaNMask; + bool preserve_nan = (get_fp_fast_math_flags_for_op(result_type, id) & drop_nan_inf) != drop_nan_inf; + const char *preserve_str = preserve_nan ? "precise" : "fast"; + + // TODO: Emit the default behavior to match existing code. Might need to be revisited. + // Only fp32 has the precise:: override. +#define EMIT_PRECISE_OVERRIDE(glsl_op, op) \ + case GLSLstd450##glsl_op: \ + if (restype.basetype == SPIRType::Float && preserve_nan) \ + emit_unary_func_op(result_type, id, args[0], "precise::" op); \ + else \ + CompilerGLSL::emit_glsl_op(result_type, id, eop, args, count); \ + break + switch (op) { + EMIT_PRECISE_OVERRIDE(Cos, "cos"); + EMIT_PRECISE_OVERRIDE(Sin, "sin"); + EMIT_PRECISE_OVERRIDE(Tan, "tan"); + EMIT_PRECISE_OVERRIDE(Acos, "acos"); + EMIT_PRECISE_OVERRIDE(Asin, "asin"); + EMIT_PRECISE_OVERRIDE(Atan, "atan"); + EMIT_PRECISE_OVERRIDE(Exp, "exp"); + EMIT_PRECISE_OVERRIDE(Exp2, "exp2"); + EMIT_PRECISE_OVERRIDE(Log, "log"); + EMIT_PRECISE_OVERRIDE(Log2, "log2"); + EMIT_PRECISE_OVERRIDE(Sqrt, "sqrt"); +#undef EMIT_PRECISE_OVERRIDE + case GLSLstd450Sinh: - emit_unary_func_op(result_type, id, args[0], "fast::sinh"); + if (restype.basetype == SPIRType::Half) + { + auto ftype = restype; + ftype.basetype = SPIRType::Float; + + // MSL does not have overload for half. Force-cast back to half. + auto expr = join(type_to_glsl(restype), "(", preserve_str, "::sinh(", type_to_glsl(ftype), "(", to_unpacked_expression(args[0]), ")))"); + emit_op(result_type, id, expr, should_forward(args[0])); + inherit_expression_dependencies(id, args[0]); + } + else if (preserve_nan) + emit_unary_func_op(result_type, id, args[0], "precise::sinh"); + else + emit_unary_func_op(result_type, id, args[0], "fast::sinh"); break; case GLSLstd450Cosh: - emit_unary_func_op(result_type, id, args[0], "fast::cosh"); + if (restype.basetype == SPIRType::Half) + { + auto ftype = restype; + ftype.basetype = SPIRType::Float; + + // MSL does not have overload for half. Force-cast back to half. + auto expr = join(type_to_glsl(restype), "(", preserve_str, "::cosh(", type_to_glsl(ftype), "(", to_unpacked_expression(args[0]), ")))"); + emit_op(result_type, id, expr, should_forward(args[0])); + inherit_expression_dependencies(id, args[0]); + } + else if (preserve_nan) + emit_unary_func_op(result_type, id, args[0], "precise::cosh"); + else + emit_unary_func_op(result_type, id, args[0], "fast::cosh"); break; case GLSLstd450Tanh: - emit_unary_func_op(result_type, id, args[0], "precise::tanh"); + if (restype.basetype == SPIRType::Half) + { + auto ftype = restype; + ftype.basetype = SPIRType::Float; + + // MSL does not have overload for half. Force-cast back to half. + auto expr = join(type_to_glsl(restype), "(", preserve_str, "::tanh(", type_to_glsl(ftype), "(", to_unpacked_expression(args[0]), ")))"); + emit_op(result_type, id, expr, should_forward(args[0])); + inherit_expression_dependencies(id, args[0]); + } + else + emit_unary_func_op(result_type, id, args[0], "precise::tanh"); break; case GLSLstd450Atan2: - emit_binary_func_op(result_type, id, args[0], args[1], "precise::atan2"); + if (restype.basetype == SPIRType::Half) + { + // MSL does not have overload for half. Force-cast back to half. + auto ftype = restype; + ftype.basetype = SPIRType::Float; + + auto expr = join(type_to_glsl(restype), + "(", preserve_str, "::atan2(", + type_to_glsl(ftype), "(", to_unpacked_expression(args[0]), "), ", + type_to_glsl(ftype), "(", to_unpacked_expression(args[1]), ")))"); + emit_op(result_type, id, expr, should_forward(args[0]) && should_forward(args[1])); + inherit_expression_dependencies(id, args[0]); + inherit_expression_dependencies(id, args[1]); + } + else + emit_binary_func_op(result_type, id, args[0], args[1], "precise::atan2"); break; case GLSLstd450InverseSqrt: - emit_unary_func_op(result_type, id, args[0], "rsqrt"); + if (restype.basetype == SPIRType::Float && preserve_nan) + emit_unary_func_op(result_type, id, args[0], "precise::rsqrt"); + else + emit_unary_func_op(result_type, id, args[0], "rsqrt"); break; case GLSLstd450RoundEven: emit_unary_func_op(result_type, id, args[0], "rint"); @@ -10435,11 +11983,14 @@ void CompilerMSL::emit_glsl_op(uint32_t result_type, uint32_t id, uint32_t eop, { auto &exp_type = expression_type(args[0]); // MSL does not support scalar versions here. - // MSL has no implementation for normalize in the fast:: namespace for half2 and half3 + // MSL has no implementation for normalize in the fast:: namespace for half // Returns -1 or 1 for valid input, sign() does the job. + + // precise::normalize asm looks ridiculous. + // Don't think this actually matters unless proven otherwise. if (exp_type.vecsize == 1) emit_unary_func_op(result_type, id, args[0], "sign"); - else if (exp_type.vecsize <= 3 && exp_type.basetype == SPIRType::Half) + else if (exp_type.basetype == SPIRType::Half) emit_unary_func_op(result_type, id, args[0], "normalize"); else emit_unary_func_op(result_type, id, args[0], "fast::normalize"); @@ -10502,7 +12053,10 @@ void CompilerMSL::emit_glsl_op(uint32_t result_type, uint32_t id, uint32_t eop, case GLSLstd450Pow: // powr makes x < 0.0 undefined, just like SPIR-V. - emit_binary_func_op(result_type, id, args[0], args[1], "powr"); + if (restype.basetype == SPIRType::Float && preserve_nan) + emit_binary_func_op(result_type, id, args[0], args[1], "precise::powr"); + else + emit_binary_func_op(result_type, id, args[0], args[1], "powr"); break; default: @@ -10596,7 +12150,7 @@ void CompilerMSL::emit_function_prototype(SPIRFunction &func, const Bitset &) decl += "thread "; decl += type_to_glsl(type); decl += " (&spvReturnValue)"; - decl += type_to_array_glsl(type); + decl += type_to_array_glsl(type, 0); if (!func.arguments.empty()) decl += ", "; } @@ -10626,6 +12180,21 @@ void CompilerMSL::emit_function_prototype(SPIRFunction &func, const Bitset &) if (ir.ids[initializer].get_type() == TypeNone || ir.ids[initializer].get_type() == TypeExpression) set(ed_var.initializer, "{}", ed_var.basetype, true); } + + // add `taskPayloadSharedEXT` variable to entry-point arguments + for (auto &v : func.local_variables) + { + auto &var = get(v); + if (var.storage != StorageClassTaskPayloadWorkgroupEXT) + continue; + + add_local_variable_name(v); + SPIRFunction::Parameter arg = {}; + arg.id = v; + arg.type = var.basetype; + arg.alias_global_variable = true; + decl += join(", ", argument_decl(arg), " [[payload]]"); + } } for (auto &arg : func.arguments) @@ -10665,7 +12234,7 @@ void CompilerMSL::emit_function_prototype(SPIRFunction &func, const Bitset &) { if (arg_type.array.empty() || (var ? is_var_runtime_size_array(*var) : is_runtime_size_array(arg_type))) { - decl += join(", ", sampler_type(arg_type, arg.id), " ", to_sampler_expression(name_id)); + decl += join(", ", sampler_type(arg_type, arg.id, false), " ", to_sampler_expression(name_id)); } else { @@ -10673,7 +12242,7 @@ void CompilerMSL::emit_function_prototype(SPIRFunction &func, const Bitset &) descriptor_address_space(name_id, StorageClassUniformConstant, "thread const"); - decl += join(", ", sampler_address_space, " ", sampler_type(arg_type, name_id), "& ", + decl += join(", ", sampler_address_space, " ", sampler_type(arg_type, name_id, false), "& ", to_sampler_expression(name_id)); } } @@ -10720,8 +12289,7 @@ string CompilerMSL::to_function_name(const TextureFunctionNameArguments &args) is_dynamic_img_sampler = has_extended_decoration(var->self, SPIRVCrossDecorationDynamicImageSampler); } - // Special-case gather. We have to alter the component being looked up - // in the swizzle case. + // Special-case gather. We have to alter the component being looked up in the swizzle case. if (msl_options.swizzle_texture_samples && args.base.is_gather && !is_dynamic_img_sampler && (!constexpr_sampler || !constexpr_sampler->ycbcr_conversion_enable)) { @@ -10730,6 +12298,15 @@ string CompilerMSL::to_function_name(const TextureFunctionNameArguments &args) return is_compare ? "spvGatherCompareSwizzle" : "spvGatherSwizzle"; } + // Special-case gather with an array of offsets. We have to lower into 4 separate gathers. + if (args.has_array_offsets && !is_dynamic_img_sampler && + (!constexpr_sampler || !constexpr_sampler->ycbcr_conversion_enable)) + { + bool is_compare = comparison_ids.count(img); + add_spv_func_and_recompile(is_compare ? SPVFuncImplGatherCompareConstOffsets : SPVFuncImplGatherConstOffsets); + return is_compare ? "spvGatherCompareConstOffsets" : "spvGatherConstOffsets"; + } + auto *combined = maybe_get(img); // Texture reference @@ -10910,6 +12487,10 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool farg_str += to_expression(combined ? combined->image : img); } + // Gathers with constant offsets call a special function, so include the texture. + if (args.has_array_offsets) + farg_str += to_expression(img); + // Sampler reference if (!args.base.is_fetch) { @@ -10926,18 +12507,24 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool used_swizzle_buffer = true; } - // Swizzled gather puts the component before the other args, to allow template - // deduction to work. - if (args.component && msl_options.swizzle_texture_samples) + // Const offsets gather puts the const offsets before the other args. + if (args.has_array_offsets) { - forward = should_forward(args.component); + forward = forward && should_forward(args.offset); + farg_str += ", " + to_unpacked_expression(args.offset); + } + + // Const offsets gather or swizzled gather puts the component before the other args. + if (args.component && (args.has_array_offsets || msl_options.swizzle_texture_samples)) + { + forward = forward && should_forward(args.component); farg_str += ", " + to_component_argument(args.component); } } // Texture coordinates forward = forward && should_forward(args.coord); - auto coord_expr = to_enclosed_expression(args.coord); + auto coord_expr = to_enclosed_unpacked_expression(args.coord); auto &coord_type = expression_type(args.coord); bool coord_is_fp = type_is_floating_point(coord_type); bool is_cube_fetch = false; @@ -11061,14 +12648,14 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool if (type.basetype != SPIRType::UInt) tex_coords += join(" + uint2(", bitcast_expression(SPIRType::UInt, args.offset), ", 0)"); else - tex_coords += join(" + uint2(", to_enclosed_expression(args.offset), ", 0)"); + tex_coords += join(" + uint2(", to_enclosed_unpacked_expression(args.offset), ", 0)"); } else { if (type.basetype != SPIRType::UInt) tex_coords += " + " + bitcast_expression(SPIRType::UInt, args.offset); else - tex_coords += " + " + to_enclosed_expression(args.offset); + tex_coords += " + " + to_enclosed_unpacked_expression(args.offset); } } @@ -11155,10 +12742,10 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool string dref_expr; if (args.base.is_proj) - dref_expr = join(to_enclosed_expression(args.dref), " / ", + dref_expr = join(to_enclosed_unpacked_expression(args.dref), " / ", to_extract_component_expression(args.coord, alt_coord_component)); else - dref_expr = to_expression(args.dref); + dref_expr = to_unpacked_expression(args.dref); if (sampling_type_needs_f32_conversion(dref_type)) dref_expr = convert_to_f32(dref_expr, 1); @@ -11209,7 +12796,7 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool if (bias && (imgtype.image.dim != Dim1D || msl_options.texture_1D_as_2D)) { forward = forward && should_forward(bias); - farg_str += ", bias(" + to_expression(bias) + ")"; + farg_str += ", bias(" + to_unpacked_expression(bias) + ")"; } // Metal does not support LOD for 1D textures. @@ -11218,7 +12805,7 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool forward = forward && should_forward(lod); if (args.base.is_fetch) { - farg_str += ", " + to_expression(lod); + farg_str += ", " + to_unpacked_expression(lod); } else if (msl_options.sample_dref_lod_array_as_grad && args.dref && imgtype.image.arrayed) { @@ -11275,12 +12862,12 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool extent = "float3(1.0)"; break; } - farg_str += join(", ", grad_opt, "(", grad_coord, "exp2(", to_expression(lod), " - 0.5) / ", extent, - ", exp2(", to_expression(lod), " - 0.5) / ", extent, ")"); + farg_str += join(", ", grad_opt, "(", grad_coord, "exp2(", to_unpacked_expression(lod), " - 0.5) / ", extent, + ", exp2(", to_unpacked_expression(lod), " - 0.5) / ", extent, ")"); } else { - farg_str += ", level(" + to_expression(lod) + ")"; + farg_str += ", level(" + to_unpacked_expression(lod) + ")"; } } else if (args.base.is_fetch && !lod && (imgtype.image.dim != Dim1D || msl_options.texture_1D_as_2D) && @@ -11326,7 +12913,7 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool grad_opt = "unsupported_gradient_dimension"; break; } - farg_str += join(", ", grad_opt, "(", grad_coord, to_expression(grad_x), ", ", to_expression(grad_y), ")"); + farg_str += join(", ", grad_opt, "(", grad_coord, to_unpacked_expression(grad_x), ", ", to_unpacked_expression(grad_y), ")"); } if (args.min_lod) @@ -11335,16 +12922,16 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool SPIRV_CROSS_THROW("min_lod_clamp() is only supported in MSL 2.2+ and up."); forward = forward && should_forward(args.min_lod); - farg_str += ", min_lod_clamp(" + to_expression(args.min_lod) + ")"; + farg_str += ", min_lod_clamp(" + to_unpacked_expression(args.min_lod) + ")"; } // Add offsets string offset_expr; const SPIRType *offset_type = nullptr; - if (args.offset && !args.base.is_fetch) + if (args.offset && !args.base.is_fetch && !args.has_array_offsets) { forward = forward && should_forward(args.offset); - offset_expr = to_expression(args.offset); + offset_expr = to_unpacked_expression(args.offset); offset_type = &expression_type(args.offset); } @@ -11380,7 +12967,7 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool } } - if (args.component) + if (args.component && !args.has_array_offsets) { // If 2D has gather component, ensure it also has an offset arg if (imgtype.image.dim == Dim2D && offset_expr.empty()) @@ -11410,7 +12997,7 @@ string CompilerMSL::to_function_args(const TextureFunctionArguments &args, bool { forward = forward && should_forward(args.sample); farg_str += ", "; - farg_str += to_expression(args.sample); + farg_str += to_unpacked_expression(args.sample); } *p_forward = forward; @@ -11689,7 +13276,7 @@ string CompilerMSL::to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_ } // Dereference pointer variables where needed. // FIXME: This dereference is actually backwards. We should really just support passing pointer variables between functions. - else if (should_dereference(id)) + else if (should_dereference_caller_param(id)) arg_str += dereference_expression(type, CompilerGLSL::to_func_call_arg(arg, id)); else arg_str += CompilerGLSL::to_func_call_arg(arg, id); @@ -11809,21 +13396,26 @@ string CompilerMSL::to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_ string CompilerMSL::to_sampler_expression(uint32_t id) { auto *combined = maybe_get(id); - auto expr = to_expression(combined ? combined->image : VariableID(id)); - auto index = expr.find_first_of('['); + if (combined && combined->sampler) + return to_expression(combined->sampler); - uint32_t samp_id = 0; - if (combined) - samp_id = combined->sampler; + uint32_t expr_id = combined ? uint32_t(combined->image) : id; - if (index == string::npos) - return samp_id ? to_expression(samp_id) : expr + sampler_name_suffix; - else + // Constexpr samplers are declared as local variables, + // so exclude any qualifier names on the image expression. + if (auto *var = maybe_get_backing_variable(expr_id)) { - auto image_expr = expr.substr(0, index); - auto array_expr = expr.substr(index); - return samp_id ? to_expression(samp_id) : (image_expr + sampler_name_suffix + array_expr); + uint32_t img_id = var->basevariable ? var->basevariable : VariableID(var->self); + if (find_constexpr_sampler(img_id)) + return Compiler::to_name(img_id) + sampler_name_suffix; } + + auto img_expr = to_expression(expr_id); + auto index = img_expr.find_first_of('['); + if (index == string::npos) + return img_expr + sampler_name_suffix; + else + return img_expr.substr(0, index) + sampler_name_suffix + img_expr.substr(index); } string CompilerMSL::to_swizzle_expression(uint32_t id) @@ -11851,7 +13443,6 @@ string CompilerMSL::to_swizzle_expression(uint32_t id) string CompilerMSL::to_buffer_size_expression(uint32_t id) { auto expr = to_expression(id); - auto index = expr.find_first_of('['); // This is quite crude, but we need to translate the reference name (*spvDescriptorSetN.name) to // the pointer expression spvDescriptorSetN.name to make a reasonable expression here. @@ -11859,17 +13450,26 @@ string CompilerMSL::to_buffer_size_expression(uint32_t id) if (expr.size() >= 3 && expr[0] == '(' && expr[1] == '*') expr = address_of_expression(expr); + auto index = expr.find_first_of('['); + string buffer_expr, array_expr; + + if (index != string::npos) + { + buffer_expr = expr.substr(0, index); + array_expr = expr.substr(index); + } + // If a buffer is part of an argument buffer translate this to a legal identifier. for (auto &c : expr) if (c == '.') c = '_'; if (index == string::npos) + { return expr + buffer_size_name_suffix; + } else { - auto buffer_expr = expr.substr(0, index); - auto array_expr = expr.substr(index); if (auto var = maybe_get_backing_variable(id)) { if (is_var_runtime_size_array(*var)) @@ -11882,6 +13482,11 @@ string CompilerMSL::to_buffer_size_expression(uint32_t id) return buffer_expr + ".length(" + array_expr.substr(1, last_pos - 1) + ")"; } } + + for (auto &c : buffer_expr) + if (c == '.') + c = '_'; + return buffer_expr + buffer_size_name_suffix + array_expr; } } @@ -11936,14 +13541,34 @@ string CompilerMSL::convert_row_major_matrix(string exp_str, const SPIRType &exp // Called automatically at the end of the entry point function void CompilerMSL::emit_fixup() { - if (is_vertex_like_shader() && stage_out_var_id && !qual_pos_var_name.empty() && !capture_output_to_buffer) + if (stage_out_var_id && !capture_output_to_buffer) { - if (options.vertex.fixup_clipspace) - statement(qual_pos_var_name, ".z = (", qual_pos_var_name, ".z + ", qual_pos_var_name, - ".w) * 0.5; // Adjust clip-space for Metal"); + if (needs_point_size_output && !writes_to_point_size) + statement(builtin_to_glsl(BuiltInPointSize, StorageClassOutput), " = ", format_float(msl_options.default_point_size), ";"); - if (options.vertex.flip_vert_y) - statement(qual_pos_var_name, ".y = -(", qual_pos_var_name, ".y);", " // Invert Y-axis for Metal"); + if (is_vertex_like_shader() && !qual_pos_var_name.empty()) + { + if (msl_options.emulate_reversed_depth_viewport) + { + if (qual_viewport_idx_var_name.empty()) + // If ViewportIndex is not written, the primitive uses viewport 0. + statement("if ((spvEmulatedReversedDepthViewportMask & 1u) != 0u)"); + else + statement("if (((spvEmulatedReversedDepthViewportMask >> uint(", qual_viewport_idx_var_name, + ")) & 1u) != 0u)"); + begin_scope(); + statement(qual_pos_var_name, ".z = ", qual_pos_var_name, ".w - ", qual_pos_var_name, + ".z; // Emulate reversed-depth viewport"); + end_scope(); + } + + if (options.vertex.fixup_clipspace) + statement(qual_pos_var_name, ".z = (", qual_pos_var_name, ".z + ", qual_pos_var_name, + ".w) * 0.5; // Adjust clip-space for Metal"); + + if (options.vertex.flip_vert_y) + statement(qual_pos_var_name, ".y = -(", qual_pos_var_name, ".y);", " // Invert Y-axis for Metal"); + } } } @@ -12057,10 +13682,62 @@ string CompilerMSL::to_struct_member(const SPIRType &type, uint32_t member_type_ ((stage_out_var_id && get_stage_out_struct_type().self == type.self && variable_storage_requires_stage_io(StorageClassOutput)) || (stage_in_var_id && get_stage_in_struct_type().self == type.self && - variable_storage_requires_stage_io(StorageClassInput))); + variable_storage_requires_stage_io(StorageClassInput))) || + is_mesh_shader(); if (is_ib_in_out && is_member_builtin(type, index, &builtin)) is_using_builtin_array = true; - array_type = type_to_array_glsl(physical_type); + array_type = type_to_array_glsl(physical_type, orig_id); + } + + if (is_mesh_shader()) + { + BuiltIn builtin = BuiltInMax; + if (is_member_builtin(type, index, &builtin)) + { + if (builtin == BuiltInPrimitiveShadingRateKHR) + { + // not supported in metal 3.0 + is_using_builtin_array = false; + return ""; + } + + SPIRType metallic_type = *declared_type; + if (builtin == BuiltInCullPrimitiveEXT) + metallic_type.basetype = SPIRType::Boolean; + else if (builtin == BuiltInPrimitiveId || builtin == BuiltInLayer || builtin == BuiltInViewportIndex) + metallic_type.basetype = SPIRType::UInt; + + is_using_builtin_array = true; + std::string result; + if (has_member_decoration(type.self, orig_id, DecorationBuiltIn)) + { + // avoid '_RESERVED_IDENTIFIER_FIXUP_' in variable name + result = join(type_to_glsl(metallic_type, orig_id, false), " ", qualifier, + builtin_to_glsl(builtin, StorageClassOutput), member_attribute_qualifier(type, index), + array_type, ";"); + } + else + { + result = join(type_to_glsl(metallic_type, orig_id, false), " ", qualifier, + to_member_name(type, index), member_attribute_qualifier(type, index), array_type, ";"); + } + is_using_builtin_array = false; + return result; + } + } + + if (orig_id) + { + auto *data_type = declared_type; + if (is_pointer(*data_type)) + data_type = &get_pointee_type(*data_type); + + if (is_array(*data_type) && get_resource_array_size(*data_type, orig_id) == 0) + { + // Hack for declaring unsized array of resources. Need to declare dummy sized array by value inline. + // This can then be wrapped in spvDescriptorArray as usual. + array_type = "[1] /* unsized array hack */"; + } } string decl_type; @@ -12080,7 +13757,29 @@ string CompilerMSL::to_struct_member(const SPIRType &type, uint32_t member_type_ else decl_type = type_to_glsl(*declared_type, orig_id, true); - auto result = join(pack_pfx, decl_type, " ", qualifier, + if (physical_type.basetype == SPIRType::Struct && + has_decoration(physical_type.self, DecorationArrayStride) && + is_array(physical_type)) + { + uint32_t native_stride = get_decoration(physical_type.self, DecorationArrayStride); + uint32_t array_stride = get_decoration(type.member_types[index], DecorationArrayStride); + auto *struct_array_type = &physical_type; + + while (struct_array_type->parent_type && is_array(get(struct_array_type->parent_type))) + { + array_stride = get_decoration(struct_array_type->parent_type, DecorationArrayStride); + struct_array_type = &get(struct_array_type->parent_type); + } + + if (array_stride != native_stride) + decl_type = join("spvPaddedArrayElement<", decl_type, ", ", array_stride, ">"); + } + + const char *overlapping_binding_tag = + has_extended_member_decoration(type.self, index, SPIRVCrossDecorationOverlappingBinding) ? + "// Overlapping binding: " : ""; + + auto result = join(overlapping_binding_tag, pack_pfx, decl_type, " ", qualifier, to_member_name(type, index), member_attribute_qualifier(type, index), array_type, ";"); is_using_builtin_array = false; @@ -12098,22 +13797,22 @@ void CompilerMSL::emit_struct_member(const SPIRType &type, uint32_t member_type_ statement("char _m", index, "_pad", "[", pad_len, "];"); } + BuiltIn builtin = BuiltInMax; + if (is_mesh_shader() && is_member_builtin(type, index, &builtin)) + { + if (!has_active_builtin(builtin, StorageClassOutput) && !has_active_builtin(builtin, StorageClassInput)) + { + // Do not emit unused builtins in mesh-output blocks + return; + } + } + // Handle HLSL-style 0-based vertex/instance index. builtin_declaration = true; statement(to_struct_member(type, member_type_id, index, qualifier)); builtin_declaration = false; } -void CompilerMSL::emit_struct_padding_target(const SPIRType &type) -{ - uint32_t struct_size = get_declared_struct_size_msl(type, true, true); - uint32_t target_size = get_extended_decoration(type.self, SPIRVCrossDecorationPaddingTarget); - if (target_size < struct_size) - SPIRV_CROSS_THROW("Cannot pad with negative bytes."); - else if (target_size > struct_size) - statement("char _m0_final_padding[", target_size - struct_size, "];"); -} - // Return a MSL qualifier for the specified function attribute member string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t index) { @@ -12153,9 +13852,6 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in return ""; return string(" [[") + builtin_qualifier(builtin) + "]]"; - case BuiltInDrawIndex: - SPIRV_CROSS_THROW("DrawIndex is not supported in MSL."); - default: return ""; } @@ -12171,9 +13867,11 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in return string(" [[attribute(") + convert_to_string(locn) + ")]]"; } - // Vertex and tessellation evaluation function outputs - if (((execution.model == ExecutionModelVertex && !msl_options.vertex_for_tessellation) || is_tese_shader()) && - type.storage == StorageClassOutput) + bool use_semantic_stage_output = is_mesh_shader() || is_tese_shader() || + (execution.model == ExecutionModelVertex && !msl_options.vertex_for_tessellation); + + // Vertex, mesh and tessellation evaluation function outputs + if ((type.storage == StorageClassOutput || is_mesh_shader()) && use_semantic_stage_output) { if (is_builtin) { @@ -12192,6 +13890,9 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in /* fallthrough */ case BuiltInPosition: case BuiltInLayer: + case BuiltInCullPrimitiveEXT: + case BuiltInPrimitiveShadingRateKHR: + case BuiltInPrimitiveId: return string(" [[") + builtin_qualifier(builtin) + "]]" + (mbr_type.array.empty() ? "" : " "); case BuiltInClipDistance: @@ -12345,17 +14046,10 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in else quals = member_location_attribute_qualifier(type, index); - if (builtin == BuiltInBaryCoordKHR || builtin == BuiltInBaryCoordNoPerspKHR) + if (builtin == BuiltInBaryCoordKHR && has_member_decoration(type.self, index, DecorationNoPerspective)) { - if (has_member_decoration(type.self, index, DecorationFlat) || - has_member_decoration(type.self, index, DecorationCentroid) || - has_member_decoration(type.self, index, DecorationSample) || - has_member_decoration(type.self, index, DecorationNoPerspective)) - { - // NoPerspective is baked into the builtin type. - SPIRV_CROSS_THROW( - "Flat, Centroid, Sample, NoPerspective decorations are not supported for BaryCoord inputs."); - } + // NoPerspective is baked into the builtin type. + SPIRV_CROSS_THROW("NoPerspective decorations are not supported for BaryCoord inputs."); } // Don't bother decorating integers with the 'flat' attribute; it's @@ -12373,6 +14067,10 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in { if (!quals.empty()) quals += ", "; + + if (builtin == BuiltInBaryCoordNoPerspKHR || builtin == BuiltInBaryCoordKHR) + SPIRV_CROSS_THROW("Centroid interpolation not supported for barycentrics in MSL."); + if (has_member_decoration(type.self, index, DecorationNoPerspective)) quals += "centroid_no_perspective"; else @@ -12382,17 +14080,27 @@ string CompilerMSL::member_attribute_qualifier(const SPIRType &type, uint32_t in { if (!quals.empty()) quals += ", "; + + if (builtin == BuiltInBaryCoordNoPerspKHR || builtin == BuiltInBaryCoordKHR) + SPIRV_CROSS_THROW("Sample interpolation not supported for barycentrics in MSL."); + if (has_member_decoration(type.self, index, DecorationNoPerspective)) quals += "sample_no_perspective"; else quals += "sample_perspective"; } - else if (has_member_decoration(type.self, index, DecorationNoPerspective)) + else if (has_member_decoration(type.self, index, DecorationNoPerspective) || builtin == BuiltInBaryCoordNoPerspKHR) { if (!quals.empty()) quals += ", "; quals += "center_no_perspective"; } + else if (builtin == BuiltInBaryCoordKHR) + { + if (!quals.empty()) + quals += ", "; + quals += "center_perspective"; + } } if (!quals.empty()) @@ -12519,7 +14227,7 @@ uint32_t CompilerMSL::get_member_location(uint32_t type_id, uint32_t index, uint return k_unknown_location; } -uint32_t CompilerMSL::get_or_allocate_builtin_input_member_location(spv::BuiltIn builtin, +uint32_t CompilerMSL::get_or_allocate_builtin_input_member_location(BuiltIn builtin, uint32_t type_id, uint32_t index, uint32_t *comp) { @@ -12564,7 +14272,7 @@ uint32_t CompilerMSL::get_or_allocate_builtin_input_member_location(spv::BuiltIn return loc; } -uint32_t CompilerMSL::get_or_allocate_builtin_output_member_location(spv::BuiltIn builtin, +uint32_t CompilerMSL::get_or_allocate_builtin_output_member_location(BuiltIn builtin, uint32_t type_id, uint32_t index, uint32_t *comp) { @@ -12607,19 +14315,38 @@ uint32_t CompilerMSL::get_or_allocate_builtin_output_member_location(spv::BuiltI return loc; } +bool CompilerMSL::entry_point_is_vertex() const +{ + // MSL vertex entrypoint is used for non-tessellation vertex stage or tessellation evaluation stage. + return (get_execution_model() == ExecutionModelVertex && !msl_options.vertex_for_tessellation) || + get_execution_model() == ExecutionModelTessellationEvaluation; +} + +bool CompilerMSL::entry_point_returns_stage_output() const +{ + if (get_execution_model() == ExecutionModelVertex && msl_options.vertex_for_tessellation) + return false; + bool ep_should_return_output = !get_is_rasterization_disabled(); + return stage_out_var_id && ep_should_return_output; +} + +bool CompilerMSL::entry_point_requires_const_device_buffers() const +{ + return !has_descriptor_side_effects_buffer && !capture_output_to_buffer; +} + // Returns the type declaration for a function, including the // entry type if the current function is the entry point function string CompilerMSL::func_type_decl(SPIRType &type) { // The regular function return type. If not processing the entry point function, that's all we need - string return_type = type_to_glsl(type) + type_to_array_glsl(type); + string return_type = type_to_glsl(type) + type_to_array_glsl(type, 0); if (!processing_entry_point) return return_type; // If an outgoing interface block has been defined, and it should be returned, override the entry point return type - bool ep_should_return_output = !get_is_rasterization_disabled(); - if (stage_out_var_id && ep_should_return_output) - return_type = type_to_glsl(get_stage_out_struct_type()) + type_to_array_glsl(type); + if (entry_point_returns_stage_output()) + return_type = type_to_glsl(get_stage_out_struct_type()) + type_to_array_glsl(type, 0); // Prepend a entry type, based on the execution model string entry_type; @@ -12655,6 +14382,12 @@ string CompilerMSL::func_type_decl(SPIRType &type) case ExecutionModelKernel: entry_type = "kernel"; break; + case ExecutionModelMeshEXT: + entry_type = "[[mesh]]"; + break; + case ExecutionModelTaskEXT: + entry_type = "[[object]]"; + break; default: entry_type = "unknown"; break; @@ -12673,6 +14406,11 @@ bool CompilerMSL::is_tese_shader() const return get_execution_model() == ExecutionModelTessellationEvaluation; } +bool CompilerMSL::is_mesh_shader() const +{ + return get_execution_model() == ExecutionModelMeshEXT; +} + bool CompilerMSL::uses_explicit_early_fragment_test() { auto &ep_flags = get_entry_point().flags; @@ -12680,12 +14418,34 @@ bool CompilerMSL::uses_explicit_early_fragment_test() } // In MSL, address space qualifiers are required for all pointer or reference variables -string CompilerMSL::get_argument_address_space(const SPIRVariable &argument) +string CompilerMSL::get_variable_address_space(const SPIRVariable &argument) { const auto &type = get(argument.basetype); return get_type_address_space(type, argument.self, true); } +string CompilerMSL::get_leaf_argument_address_space(const SPIRVariable &argument) +{ + const auto &type = get(argument.basetype); + // BDA and variable buffer pointer is always passed around by (pointer) value. There is no storage class for the argument itself. + if (is_physical_or_buffer_pointer(type)) + return ""; + return get_type_address_space(type, argument.self, true); +} + +bool CompilerMSL::decoration_flags_signal_volatile(const Bitset &flags) const +{ + // Using volatile for coherent pre-3.2 is definitely not correct, but it's something. + // MSL 3.2 adds actual coherent qualifiers. + return flags.get(DecorationVolatile) || + (flags.get(DecorationCoherent) && !msl_options.supports_msl_version(3, 2)); +} + +bool CompilerMSL::decoration_flags_signal_coherent(const Bitset &flags) const +{ + return flags.get(DecorationCoherent) && msl_options.supports_msl_version(3, 2); +} + string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bool argument) { // This can be called for variable pointer contexts as well, so be very careful about which method we choose. @@ -12695,8 +14455,17 @@ string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bo (has_decoration(type.self, DecorationBlock) || has_decoration(type.self, DecorationBufferBlock))) flags = get_buffer_block_flags(id); else + { flags = get_decoration_bitset(id); + if (type.basetype == SPIRType::Struct && + (has_decoration(type.self, DecorationBlock) || + has_decoration(type.self, DecorationBufferBlock))) + { + flags.merge_or(ir.get_buffer_block_type_flags(type)); + } + } + const char *addr_space = nullptr; switch (type.storage) { @@ -12707,13 +14476,14 @@ string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bo case StorageClassStorageBuffer: case StorageClassPhysicalStorageBuffer: { - // For arguments from variable pointers, we use the write count deduction, so - // we should not assume any constness here. Only for global SSBOs. - bool readonly = false; - if (!var || has_decoration(type.self, DecorationBlock)) - readonly = flags.get(DecorationNonWritable); - - addr_space = readonly ? "const device" : "device"; + // When dealing with descriptor aliasing, it becomes very problematic to make use of + // readonly qualifiers. + // If rasterization is not disabled in vertex/tese, Metal does not allow side effects and refuses to compile "device", + // even if there are no writes. Just force const device. + if (entry_point_requires_const_device_buffers() && type.basetype != SPIRType::AtomicCounter) + addr_space = "const device"; + else + addr_space = "device"; break; } @@ -12724,12 +14494,18 @@ string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bo { bool ssbo = has_decoration(type.self, DecorationBufferBlock); if (ssbo) - addr_space = flags.get(DecorationNonWritable) ? "const device" : "device"; + { + if (entry_point_requires_const_device_buffers() && type.basetype != SPIRType::AtomicCounter) + addr_space = "const device"; + else + addr_space = "device"; + } else addr_space = "constant"; } else if (!argument) { + // This is used for helper UBOs we insert ourselves. addr_space = "constant"; } else if (type_is_msl_framebuffer_fetch(type)) @@ -12737,6 +14513,7 @@ string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bo // Subpass inputs are passed around by value. addr_space = ""; } + break; case StorageClassFunction: @@ -12780,22 +14557,44 @@ string CompilerMSL::get_type_address_space(const SPIRType &type, uint32_t id, bo addr_space = "threadgroup"; } + // BlockIO is passed as thread and lowered on return from main. + if (get_execution_model() == ExecutionModelVertex && has_decoration(type.self, DecorationBlock)) + addr_space = "thread"; + if (!addr_space) addr_space = "device"; } + + if (is_mesh_shader()) + addr_space = "threadgroup"; + break; + + case StorageClassTaskPayloadWorkgroupEXT: + if (is_mesh_shader()) + addr_space = "const object_data"; + else + addr_space = "object_data"; break; default: break; } + if (!addr_space && var && is_var_runtime_size_array(*var)) + addr_space = "device"; + if (!addr_space) { // No address space for plain values. addr_space = type.pointer || (argument && type.basetype == SPIRType::ControlPointArray) ? "thread" : ""; } - return join(flags.get(DecorationVolatile) || flags.get(DecorationCoherent) ? "volatile " : "", addr_space); + if (decoration_flags_signal_coherent(flags) && strcmp(addr_space, "device") == 0) + return join("coherent device"); + else if (decoration_flags_signal_volatile(flags) && strcmp(addr_space, "thread") != 0) + return join("volatile ", addr_space); + else + return addr_space; } const char *CompilerMSL::to_restrict(uint32_t id, bool space) @@ -12901,6 +14700,9 @@ bool CompilerMSL::is_direct_input_builtin(BuiltIn bi_type) /* fallthrough */ case BuiltInSubgroupLocalInvocationId: return !msl_options.emulate_subgroups; + case BuiltInDrawIndex: + // Emulated + return false; default: return true; } @@ -13012,6 +14814,15 @@ void CompilerMSL::entry_point_args_builtin(string &ep_args) if (needs_base_instance_arg == TriState::Yes) ep_args += built_in_func_arg(BuiltInBaseInstance, !ep_args.empty()); + if (msl_options.emulate_reversed_depth_viewport && stage_out_var_id && !capture_output_to_buffer && + is_vertex_like_shader() && !qual_pos_var_name.empty()) + { + if (!ep_args.empty()) + ep_args += ", "; + ep_args += join("constant uint& spvEmulatedReversedDepthViewportMask [[buffer(", + msl_options.reversed_depth_viewport_buffer_index, ")]]"); + } + if (capture_output_to_buffer) { // Add parameters to hold the indirect draw parameters and the shader output. This has to be handled @@ -13181,6 +14992,20 @@ void CompilerMSL::entry_point_args_builtin(string &ep_args) " [[buffer(", convert_to_string(msl_options.shader_input_buffer_index), ")]]"); } } + + if (is_mesh_shader()) + { + if (!ep_args.empty()) + ep_args += ", "; + ep_args += join("spvMesh_t spvMesh"); + } + + if (get_execution_model() == ExecutionModelTaskEXT) + { + if (!ep_args.empty()) + ep_args += ", "; + ep_args += join("mesh_grid_properties spvMgp"); + } } string CompilerMSL::entry_point_args_argument_buffer(bool append_comma) @@ -13221,7 +15046,13 @@ string CompilerMSL::entry_point_args_argument_buffer(bool append_comma) claimed_bindings.set(buffer_binding); - ep_args += get_argument_address_space(var) + " " + type_to_glsl(type) + "& " + to_restrict(id, true) + to_name(id); + ep_args += get_variable_address_space(var) + " "; + + if (recursive_inputs.count(type.self)) + ep_args += string("void* ") + to_restrict(id, true) + to_name(id) + "_vp"; + else + ep_args += type_to_glsl(type) + "& " + to_restrict(id, true) + to_name(id); + ep_args += " [[buffer(" + convert_to_string(buffer_binding) + ")]]"; next_metal_resource_index_buffer = max(next_metal_resource_index_buffer, buffer_binding + 1); @@ -13266,7 +15097,7 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) struct Resource { SPIRVariable *var; - SPIRVariable *descriptor_alias; + SPIRVariable *discrete_descriptor_alias; string name; SPIRType::BaseType basetype; uint32_t index; @@ -13283,36 +15114,60 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) !is_hidden_variable(var)) { auto &type = get_variable_data_type(var); + uint32_t desc_set = get_decoration(var_id, DecorationDescriptorSet); if (is_supported_argument_buffer_type(type) && var.storage != StorageClassPushConstant) { - uint32_t desc_set = get_decoration(var_id, DecorationDescriptorSet); if (descriptor_set_is_argument_buffer(desc_set)) + { + if (is_var_runtime_size_array(var)) + { + // Runtime arrays need to be wrapped in spvDescriptorArray from argument buffer payload. + entry_point_bindings.push_back(&var); + // We'll wrap this, so to_name() will always use non-qualified name. + // We'll need the qualified name to create temporary variable instead. + ir.meta[var_id].decoration.qualified_alias_explicit_override = true; + } return; + } } - // Handle descriptor aliasing. We can handle aliasing of buffers by casting pointers, - // but not for typed resources. - SPIRVariable *descriptor_alias = nullptr; - if (var.storage == StorageClassUniform || var.storage == StorageClassStorageBuffer) + // Handle descriptor aliasing of simple discrete cases. + // We can handle aliasing of buffers by casting pointers. + // The amount of aliasing we can perform for discrete descriptors is very limited. + // For fully mutable-style aliasing, we need argument buffers where we can exploit the fact + // that descriptors are all 8 bytes. + SPIRVariable *discrete_descriptor_alias = nullptr; + + const auto resource_is_aliasing_candidate = [this](const SPIRVariable &var_) { + return is_var_runtime_size_array(var_) || var_.storage == StorageClassUniform || + var_.storage == StorageClassStorageBuffer; + }; + + if (resource_is_aliasing_candidate(var)) { for (auto &resource : resources) { - if (get_decoration(resource.var->self, DecorationDescriptorSet) == + if (resource_is_aliasing_candidate(*resource.var) && + get_decoration(resource.var->self, DecorationDescriptorSet) == get_decoration(var_id, DecorationDescriptorSet) && get_decoration(resource.var->self, DecorationBinding) == - get_decoration(var_id, DecorationBinding) && - resource.basetype == SPIRType::Struct && type.basetype == SPIRType::Struct && - (resource.var->storage == StorageClassUniform || - resource.var->storage == StorageClassStorageBuffer)) + get_decoration(var_id, DecorationBinding)) { - descriptor_alias = resource.var; + discrete_descriptor_alias = resource.var; // Self-reference marks that we should declare the resource, // and it's being used as an alias (so we can emit void* instead). - resource.descriptor_alias = resource.var; + resource.discrete_descriptor_alias = resource.var; // Need to promote interlocked usage so that the primary declaration is correct. if (interlocked_resources.count(var_id)) interlocked_resources.insert(resource.var->self); + + // Aliasing with unroll just gets too messy to deal with. I sure hope this never comes up ... + if ((is_array(get_variable_data_type(*resource.var)) && !is_var_runtime_size_array(*resource.var)) || + (is_array(get_variable_data_type(var)) && !is_var_runtime_size_array(var))) + { + SPIRV_CROSS_THROW("Attempting to alias same binding with a descriptor array which is not implemented through argument buffers. This is unsupported."); + } break; } } @@ -13346,13 +15201,13 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) entry_point_bindings.push_back(&var); for (uint32_t i = 0; i < plane_count; i++) - resources.push_back({ &var, descriptor_alias, to_name(var_id), SPIRType::Image, - get_metal_resource_index(var, SPIRType::Image, i), i, secondary_index }); + resources.push_back({&var, discrete_descriptor_alias, to_name(var_id), SPIRType::Image, + get_metal_resource_index(var, SPIRType::Image, i), i, secondary_index }); if (type.image.dim != DimBuffer && !constexpr_sampler) { - resources.push_back({ &var, descriptor_alias, to_sampler_expression(var_id), SPIRType::Sampler, - get_metal_resource_index(var, SPIRType::Sampler), 0, 0 }); + resources.push_back({&var, discrete_descriptor_alias, to_sampler_expression(var_id), SPIRType::Sampler, + get_metal_resource_index(var, SPIRType::Sampler), 0, 0 }); } } else if (!constexpr_sampler) @@ -13362,12 +15217,12 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) // Don't allocate resource indices for aliases. uint32_t resource_index = ~0u; - if (!descriptor_alias) + if (!discrete_descriptor_alias) resource_index = get_metal_resource_index(var, type.basetype); entry_point_bindings.push_back(&var); - resources.push_back({ &var, descriptor_alias, to_name(var_id), type.basetype, - resource_index, 0, secondary_index }); + resources.push_back({&var, discrete_descriptor_alias, to_name(var_id), type.basetype, + resource_index, 0, secondary_index }); } } }); @@ -13383,6 +15238,57 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) uint32_t var_id = var.self; + if (is_var_runtime_size_array(var)) + { + add_spv_func_and_recompile(SPVFuncImplVariableDescriptorArray); + const bool ssbo = has_decoration(type.self, DecorationBufferBlock); + if ((var.storage == StorageClassStorageBuffer || ssbo) && msl_options.runtime_array_rich_descriptor) + add_spv_func_and_recompile(SPVFuncImplVariableSizedDescriptor); + else + add_spv_func_and_recompile(SPVFuncImplVariableDescriptor); + } + + if (r.discrete_descriptor_alias) + { + if (r.var == r.discrete_descriptor_alias) + { + auto primary_name = join("spvBufferAliasSet", + get_decoration(var_id, DecorationDescriptorSet), + "Binding", + get_decoration(var_id, DecorationBinding)); + + // Declare the primary alias as void* + if (!ep_args.empty()) + ep_args += ", "; + ep_args += get_variable_address_space(var) + " void* " + primary_name; + ep_args += " [[buffer(" + convert_to_string(r.index) + ")"; + if (interlocked_resources.count(var_id)) + ep_args += ", raster_order_group(0)"; + ep_args += "]]"; + } + + buffer_aliases_discrete.push_back(r.var->self); + continue; + } + + uint32_t desc_set = get_decoration(var_id, DecorationDescriptorSet); + uint32_t desc_binding = get_decoration(var_id, DecorationBinding); + + if (is_var_runtime_size_array(var)) + { + // This must be implemented as an argument buffer. Cast to intended descriptor array type on-demand. + if (!ep_args.empty()) + ep_args += ", "; + ep_args += join("device const void* spvDescriptorSet", desc_set, "Binding", desc_binding); + if (type.basetype == SPIRType::SampledImage && r.basetype == SPIRType::Sampler) + ep_args += "Smplr"; + ep_args += " [[buffer(" + convert_to_string(r.index) + ")"; + if (interlocked_resources.count(var_id)) + ep_args += ", raster_order_group(0)"; + ep_args += "]]"; + continue; + } + switch (r.basetype) { case SPIRType::Struct: @@ -13391,79 +15297,31 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) if (m.members.size() == 0) break; - if (r.descriptor_alias) - { - if (r.var == r.descriptor_alias) - { - auto primary_name = join("spvBufferAliasSet", - get_decoration(var_id, DecorationDescriptorSet), - "Binding", - get_decoration(var_id, DecorationBinding)); - - // Declare the primary alias as void* - if (!ep_args.empty()) - ep_args += ", "; - ep_args += get_argument_address_space(var) + " void* " + primary_name; - ep_args += " [[buffer(" + convert_to_string(r.index) + ")"; - if (interlocked_resources.count(var_id)) - ep_args += ", raster_order_group(0)"; - ep_args += "]]"; - } - - buffer_aliases_discrete.push_back(r.var->self); - } - else if (!type.array.empty()) + if (!type.array.empty()) { if (type.array.size() > 1) SPIRV_CROSS_THROW("Arrays of arrays of buffers are not supported."); is_using_builtin_array = true; - if (is_var_runtime_size_array(var)) + uint32_t array_size = get_resource_array_size(type, var_id); + for (uint32_t i = 0; i < array_size; ++i) { - add_spv_func_and_recompile(SPVFuncImplVariableDescriptorArray); if (!ep_args.empty()) ep_args += ", "; - const bool ssbo = has_decoration(type.self, DecorationBufferBlock); - if ((var.storage == spv::StorageClassStorageBuffer || ssbo) && - msl_options.runtime_array_rich_descriptor) - { - add_spv_func_and_recompile(SPVFuncImplVariableSizedDescriptor); - ep_args += "const device spvBufferDescriptor<" + get_argument_address_space(var) + " " + - type_to_glsl(type) + "*>* "; - } - else - { - ep_args += "const device spvDescriptor<" + get_argument_address_space(var) + " " + - type_to_glsl(type) + "*>* "; - } - ep_args += to_restrict(var_id, true) + r.name + "_"; - ep_args += " [[buffer(" + convert_to_string(r.index) + ")"; + ep_args += get_variable_address_space(var) + " " + type_to_glsl(type) + "* " + + to_restrict(var_id, true) + r.name + "_" + convert_to_string(i); + ep_args += " [[buffer(" + convert_to_string(r.index + i) + ")"; if (interlocked_resources.count(var_id)) ep_args += ", raster_order_group(0)"; ep_args += "]]"; } - else - { - uint32_t array_size = get_resource_array_size(type, var_id); - for (uint32_t i = 0; i < array_size; ++i) - { - if (!ep_args.empty()) - ep_args += ", "; - ep_args += get_argument_address_space(var) + " " + type_to_glsl(type) + "* " + - to_restrict(var_id, true) + r.name + "_" + convert_to_string(i); - ep_args += " [[buffer(" + convert_to_string(r.index + i) + ")"; - if (interlocked_resources.count(var_id)) - ep_args += ", raster_order_group(0)"; - ep_args += "]]"; - } - } is_using_builtin_array = false; } else { if (!ep_args.empty()) ep_args += ", "; - ep_args += get_argument_address_space(var) + " "; + ep_args += get_variable_address_space(var) + " "; if (recursive_inputs.count(type.self)) ep_args += string("void* ") + to_restrict(var_id, true) + r.name + "_vp"; @@ -13480,11 +15338,8 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) case SPIRType::Sampler: if (!ep_args.empty()) ep_args += ", "; - ep_args += sampler_type(type, var_id) + " " + r.name; - if (is_var_runtime_size_array(var)) - ep_args += "_ [[buffer(" + convert_to_string(r.index) + ")]]"; - else - ep_args += " [[sampler(" + convert_to_string(r.index) + ")]]"; + ep_args += sampler_type(type, var_id, false) + " " + r.name; + ep_args += " [[sampler(" + convert_to_string(r.index) + ")]]"; break; case SPIRType::Image: { @@ -13495,14 +15350,11 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) const auto &basetype = get(var.basetype); if (!type_is_msl_framebuffer_fetch(basetype)) { - ep_args += image_type_glsl(type, var_id) + " " + r.name; + ep_args += image_type_glsl(type, var_id, false) + " " + r.name; if (r.plane > 0) ep_args += join(plane_name_suffix, r.plane); - if (is_var_runtime_size_array(var)) - ep_args += "_ [[buffer(" + convert_to_string(r.index) + ")"; - else - ep_args += " [[texture(" + convert_to_string(r.index) + ")"; + ep_args += " [[texture(" + convert_to_string(r.index) + ")"; if (interlocked_resources.count(var_id)) ep_args += ", raster_order_group(0)"; @@ -13512,14 +15364,16 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) { if (msl_options.is_macos() && !msl_options.supports_msl_version(2, 3)) SPIRV_CROSS_THROW("Framebuffer fetch on Mac is not supported before MSL 2.3."); - ep_args += image_type_glsl(type, var_id) + " " + r.name; + ep_args += image_type_glsl(type, var_id, false) + " " + r.name; ep_args += " [[color(" + convert_to_string(r.index) + ")]]"; } // Emulate texture2D atomic operations if (atomic_image_vars_emulated.count(var.self)) { - ep_args += ", device atomic_" + type_to_glsl(get(basetype.image.type), 0); + auto &flags = ir.get_decoration_bitset(var.self); + const char *cv_flags = decoration_flags_signal_volatile(flags) ? "volatile " : ""; + ep_args += join(", ", cv_flags, "device atomic_", type_to_glsl(get(basetype.image.type), 0)); ep_args += "* " + r.name + "_atomic"; ep_args += " [[buffer(" + convert_to_string(r.secondary_index) + ")"; if (interlocked_resources.count(var_id)) @@ -13530,23 +15384,10 @@ void CompilerMSL::entry_point_args_discrete_descriptors(string &ep_args) } case SPIRType::AccelerationStructure: { - if (is_var_runtime_size_array(var)) - { - add_spv_func_and_recompile(SPVFuncImplVariableDescriptor); - const auto &parent_type = get(type.parent_type); - if (!ep_args.empty()) - ep_args += ", "; - ep_args += "const device spvDescriptor<" + type_to_glsl(parent_type) + ">* " + - to_restrict(var_id, true) + r.name + "_"; - ep_args += " [[buffer(" + convert_to_string(r.index) + ")]]"; - } - else - { - if (!ep_args.empty()) - ep_args += ", "; - ep_args += type_to_glsl(type, var_id) + " " + r.name; - ep_args += " [[buffer(" + convert_to_string(r.index) + ")]]"; - } + if (!ep_args.empty()) + ep_args += ", "; + ep_args += type_to_glsl(type, var_id) + " " + r.name; + ep_args += " [[buffer(" + convert_to_string(r.index) + ")]]"; break; } default: @@ -13597,6 +15438,14 @@ void CompilerMSL::fix_up_shader_inputs_outputs() }); } + if (is_mesh_shader()) + { + // If shader doesn't call SetMeshOutputsEXT, nothing should be rendered. + // No need to barrier after this, because only thread 0 writes to this later. + entry_func.fixup_hooks_in.push_back([this]() { statement("if (gl_LocalInvocationIndex == 0) spvMeshSizes.y = 0u;"); }); + entry_func.fixup_hooks_out.push_back([this]() { emit_mesh_outputs(); }); + } + // Look for sampled images and buffer. Add hooks to set up the swizzle constants or array lengths. ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { auto &type = get_variable_data_type(var); @@ -13644,7 +15493,7 @@ void CompilerMSL::fix_up_shader_inputs_outputs() statement("constant uint", is_array_type ? "* " : "& ", to_buffer_size_expression(var_id), is_array_type ? " = &" : " = ", to_name(argument_buffer_ids[desc_set]), ".spvBufferSizeConstants", "[", - convert_to_string(get_metal_resource_index(var, SPIRType::Image)), "];"); + convert_to_string(get_metal_resource_index(var, SPIRType::UInt)), "];"); } else { @@ -13657,13 +15506,14 @@ void CompilerMSL::fix_up_shader_inputs_outputs() } } - if (msl_options.replace_recursive_inputs && type_contains_recursion(type) && + if (!msl_options.argument_buffers && + msl_options.replace_recursive_inputs && type_contains_recursion(type) && (var.storage == StorageClassUniform || var.storage == StorageClassUniformConstant || var.storage == StorageClassPushConstant || var.storage == StorageClassStorageBuffer)) { recursive_inputs.insert(type.self); entry_func.fixup_hooks_in.push_back([this, &type, &var, var_id]() { - auto addr_space = get_argument_address_space(var); + auto addr_space = get_variable_address_space(var); auto var_name = to_name(var_id); statement(addr_space, " auto& ", to_restrict(var_id, true), var_name, " = *(", addr_space, " ", type_to_glsl(type), "*)", var_name, "_vp;"); @@ -14160,21 +16010,44 @@ void CompilerMSL::fix_up_shader_inputs_outputs() to_expression(builtin_dispatch_base_id), ".y;"); }); break; + case BuiltInDrawIndex: + entry_func.fixup_hooks_in.push_back([=]() { + statement(builtin_type_decl(bi_type), " ", to_expression(var_id), " = *", + to_expression(draw_index_buffer_id), ";"); + }); + break; default: break; } } else if (var.storage == StorageClassOutput && get_execution_model() == ExecutionModelFragment && - is_builtin_variable(var) && active_output_builtins.get(bi_type) && - bi_type == BuiltInSampleMask && has_additional_fixed_sample_mask()) + is_builtin_variable(var) && active_output_builtins.get(bi_type)) { - // If the additional fixed sample mask was set, we need to adjust the sample_mask - // output to reflect that. If the shader outputs the sample_mask itself too, we need - // to AND the two masks to get the final one. - string op_str = does_shader_write_sample_mask ? " &= " : " = "; - entry_func.fixup_hooks_out.push_back([=]() { - statement(to_expression(builtin_sample_mask_id), op_str, additional_fixed_sample_mask_str(), ";"); - }); + switch (bi_type) + { + case BuiltInSampleMask: + if (has_additional_fixed_sample_mask()) + { + // If the additional fixed sample mask was set, we need to adjust the sample_mask + // output to reflect that. If the shader outputs the sample_mask itself too, we need + // to AND the two masks to get the final one. + string op_str = does_shader_write_sample_mask ? " &= " : " = "; + entry_func.fixup_hooks_out.push_back([=]() { + statement(to_expression(builtin_sample_mask_id), op_str, additional_fixed_sample_mask_str(), ";"); + }); + } + break; + case BuiltInFragDepth: + if (msl_options.input_attachment_is_ds_attachment && !writes_to_depth) + { + entry_func.fixup_hooks_out.push_back([=]() { + statement(to_expression(builtin_frag_depth_id), " = ", to_expression(builtin_frag_coord_id), ".z;"); + }); + } + break; + default: + break; + } } }); } @@ -14302,24 +16175,6 @@ bool CompilerMSL::type_is_msl_framebuffer_fetch(const SPIRType &type) const msl_options.use_framebuffer_fetch_subpasses; } -bool CompilerMSL::type_is_pointer(const SPIRType &type) const -{ - if (!type.pointer) - return false; - auto &parent_type = get(type.parent_type); - // Safeguards when we forget to set pointer_depth (there is an assert for it in type_to_glsl), - // but the extra check shouldn't hurt. - return (type.pointer_depth > parent_type.pointer_depth) || !parent_type.pointer; -} - -bool CompilerMSL::type_is_pointer_to_pointer(const SPIRType &type) const -{ - if (!type.pointer) - return false; - auto &parent_type = get(type.parent_type); - return type.pointer_depth > parent_type.pointer_depth && type_is_pointer(parent_type); -} - const char *CompilerMSL::descriptor_address_space(uint32_t id, StorageClass storage, const char *plain_address_space) const { if (msl_options.argument_buffers) @@ -14356,22 +16211,35 @@ const char *CompilerMSL::descriptor_address_space(uint32_t id, StorageClass stor string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) { auto &var = get(arg.id); - auto &type = get_variable_data_type(var); auto &var_type = get(arg.type); StorageClass type_storage = var_type.storage; + // Physical pointer types are passed by pointer, not reference. + auto &data_type = get_variable_data_type(var); + bool passed_by_value = arg.alias_global_variable ? false : is_physical_or_buffer_pointer(var_type); + auto &type = passed_by_value ? var_type : data_type; + // If we need to modify the name of the variable, make sure we use the original variable. // Our alias is just a shadow variable. uint32_t name_id = var.self; if (arg.alias_global_variable && var.basevariable) name_id = var.basevariable; - bool constref = !arg.alias_global_variable && is_pointer(var_type) && arg.write_count == 0; + bool constref = !arg.alias_global_variable && !passed_by_value && is_pointer(var_type) && arg.write_count == 0; // Framebuffer fetch is plain value, const looks out of place, but it is not wrong. - if (type_is_msl_framebuffer_fetch(type)) + // readonly coming from glslang is not reliable in all cases. + // For UBOs, readonly is implied, and for SSBOs we use global check. + if (type_is_msl_framebuffer_fetch(type) || + type_storage == StorageClassStorageBuffer || + type_storage == StorageClassUniform || + type_storage == StorageClassPhysicalStorageBuffer) + { constref = false; + } else if (type_storage == StorageClassUniformConstant) + { constref = true; + } bool type_is_image = type.basetype == SPIRType::Image || type.basetype == SPIRType::SampledImage || type.basetype == SPIRType::Sampler; @@ -14391,13 +16259,13 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) spv_function_implementations.count(SPVFuncImplDynamicImageSampler); // Allow Metal to use the array template to make arrays a value type - string address_space = get_argument_address_space(var); + string address_space = arg.alias_global_variable ? get_variable_address_space(var) : get_leaf_argument_address_space(var); bool builtin = has_decoration(var.self, DecorationBuiltIn); auto builtin_type = BuiltIn(get_decoration(arg.id, DecorationBuiltIn)); if (var.basevariable && (var.basevariable == stage_in_ptr_var_id || var.basevariable == stage_out_ptr_var_id)) decl = join(cv_qualifier, type_to_glsl(type, arg.id)); - else if (builtin) + else if (builtin && !is_mesh_shader()) { // Only use templated array for Clip/Cull distance when feasible. // In other scenarios, we need need to override array length for tess levels (if used as outputs), @@ -14449,7 +16317,7 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) else { // The type is a pointer type we need to emit cv_qualifier late. - if (type_is_pointer(type)) + if (is_pointer(data_type)) { decl = type_to_glsl(type, arg.id); if (*cv_qualifier != '\0') @@ -14461,8 +16329,8 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) } } - if (!builtin && !is_pointer(var_type) && - (type_storage == StorageClassFunction || type_storage == StorageClassGeneric)) + if (passed_by_value || (!builtin && !is_pointer(var_type) && + (type_storage == StorageClassFunction || type_storage == StorageClassGeneric))) { // If the argument is a pure value and not an opaque type, we will pass by value. if (msl_options.force_native_arrays && is_array(type)) @@ -14484,14 +16352,33 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) } decl += to_expression(name_id); decl += ")"; - decl += type_to_array_glsl(type); + decl += type_to_array_glsl(type, name_id); } else { - if (!address_space.empty()) - decl = join(address_space, " ", decl); - decl += " "; - decl += to_expression(name_id); + // Variable pointer to array is kinda awkward ... + bool pointer_to_logical_buffer_array = + !is_physical_pointer(type) && is_pointer(type) && + has_decoration(type.parent_type, DecorationArrayStride); + + if (pointer_to_logical_buffer_array) + { + decl.pop_back(); + decl += " (*"; + decl += to_expression(name_id); + decl += ")"; + bool old_is_using_builtin_array = is_using_builtin_array; + is_using_builtin_array = true; + decl += type_to_array_glsl(type, name_id); + is_using_builtin_array = old_is_using_builtin_array; + } + else + { + if (!address_space.empty()) + decl = join(address_space, " ", decl); + decl += " "; + decl += to_expression(name_id); + } } } else if (is_array(type) && !type_is_image) @@ -14500,11 +16387,15 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) if (!address_space.empty()) decl = join(address_space, " ", decl); - const char *argument_buffer_space = descriptor_address_space(name_id, type_storage, nullptr); - if (argument_buffer_space) + // spvDescriptorArray absorbs the address space inside the template. + if (!is_var_runtime_size_array(var)) { - decl += " "; - decl += argument_buffer_space; + const char *argument_buffer_space = descriptor_address_space(name_id, type_storage, nullptr); + if (argument_buffer_space) + { + decl += " "; + decl += argument_buffer_space; + } } // Special case, need to override the array size here if we're using tess level as an argument. @@ -14531,7 +16422,7 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) } else { - auto array_size_decl = type_to_array_glsl(type); + auto array_size_decl = type_to_array_glsl(type, name_id); if (array_size_decl.empty()) decl += "& "; else @@ -14559,7 +16450,7 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) // for the reference has to go before the '&', but after the '*'. if (!address_space.empty()) { - if (type_is_pointer(type)) + if (is_pointer(data_type)) { if (*cv_qualifier == '\0') decl += ' '; @@ -14568,6 +16459,7 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) else decl = join(address_space, " ", decl); } + decl += "&"; decl += " "; decl += to_restrict(name_id, true); @@ -14610,7 +16502,9 @@ string CompilerMSL::argument_decl(const SPIRFunction::Parameter &arg) auto *backing_var = maybe_get_backing_variable(name_id); if (backing_var && atomic_image_vars_emulated.count(backing_var->self)) { - decl += ", device atomic_" + type_to_glsl(get(var_type.image.type), 0); + auto &flags = ir.get_decoration_bitset(backing_var->self); + const char *cv_flags = decoration_flags_signal_volatile(flags) ? "volatile " : ""; + decl += join(", ", cv_flags, "device atomic_", type_to_glsl(get(var_type.image.type), 0)); decl += "* " + to_expression(name_id) + "_atomic"; } @@ -14626,7 +16520,7 @@ string CompilerMSL::to_name(uint32_t id, bool allow_alias) const if (current_function && (current_function->self == ir.default_entry_point)) { auto *m = ir.find_meta(id); - if (m && !m->decoration.qualified_alias.empty()) + if (m && !m->decoration.qualified_alias_explicit_override && !m->decoration.qualified_alias.empty()) return m->decoration.qualified_alias; } return Compiler::to_name(id, allow_alias); @@ -14791,6 +16685,7 @@ const std::unordered_set &CompilerMSL::get_reserved_keyword_set() "quad_broadcast", "thread", "threadgroup", + "signed", }; return keywords; @@ -14800,11 +16695,16 @@ const std::unordered_set &CompilerMSL::get_illegal_func_names() { static const unordered_set illegal_func_names = { "main", + "fragment", + "vertex", + "kernel", "saturate", "assert", "fmin3", "fmax3", "divide", + "fmod", + "median3", "VARIABLE_TRACEPOINT", "STATIC_DATA_TRACEPOINT", "STATIC_DATA_TRACEPOINT_V", @@ -14923,6 +16823,13 @@ const std::unordered_set &CompilerMSL::get_illegal_func_names() "M_2_SQRTPI", "M_SQRT2", "M_SQRT1_2", + "int8", + "uint8", + "int16", + "uint16", + "float8", + "float16", + "signed", }; return illegal_func_names; @@ -15026,6 +16933,9 @@ string CompilerMSL::to_qualifiers_glsl(uint32_t id) auto *var = maybe_get(id); auto &type = expression_type(id); + if (type.storage == StorageClassTaskPayloadWorkgroupEXT) + quals += "object_data "; + if (type.storage == StorageClassWorkgroup || (var && variable_decl_is_remapped_storage(*var, StorageClassWorkgroup))) quals += "threadgroup "; @@ -15049,18 +16959,38 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) auto type_address_space = get_type_address_space(type, id); const auto *p_parent_type = &get(type.parent_type); + // If we're wrapping buffer descriptors in a spvDescriptorArray, we'll have to handle it as a special case. + if (member && id) + { + auto &var = get(id); + if (is_var_runtime_size_array(var) && is_runtime_size_array(*p_parent_type)) + { + const bool ssbo = has_decoration(p_parent_type->self, DecorationBufferBlock); + bool buffer_desc = + (var.storage == StorageClassStorageBuffer || ssbo) && + msl_options.runtime_array_rich_descriptor; + + const char *wrapper_type = buffer_desc ? "spvBufferDescriptor" : "spvDescriptor"; + add_spv_func_and_recompile(SPVFuncImplVariableDescriptorArray); + add_spv_func_and_recompile(buffer_desc ? SPVFuncImplVariableSizedDescriptor : SPVFuncImplVariableDescriptor); + + type_name = join(wrapper_type, "<", type_address_space, " ", type_to_glsl(*p_parent_type, id), " *>"); + return type_name; + } + } + // Work around C pointer qualifier rules. If glsl_type is a pointer type as well // we'll need to emit the address space to the right. // We could always go this route, but it makes the code unnatural. // Prefer emitting thread T *foo over T thread* foo since it's more readable, // but we'll have to emit thread T * thread * T constant bar; for example. - if (type_is_pointer_to_pointer(type)) + if (is_pointer(type) && is_pointer(*p_parent_type)) type_name = join(type_to_glsl(*p_parent_type, id), " ", type_address_space, " "); else { // Since this is not a pointer-to-pointer, ensure we've dug down to the base type. // Some situations chain pointers even though they are not formally pointers-of-pointers. - while (type_is_pointer(*p_parent_type)) + while (is_pointer(*p_parent_type)) p_parent_type = &get(p_parent_type->parent_type); // If we're emitting BDA, just use the templated type. @@ -15068,8 +16998,11 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) // the C-style nesting works right. // FIXME: This is somewhat of a hack. bool old_is_using_builtin_array = is_using_builtin_array; + bool pointer_to_buffer_array = is_pointer(type) && has_decoration(type.parent_type, DecorationArrayStride); if (is_physical_pointer(type)) is_using_builtin_array = false; + else if (pointer_to_buffer_array) + is_using_builtin_array = true; type_name = join(type_address_space, " ", type_to_glsl(*p_parent_type, id)); @@ -15097,6 +17030,48 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) return type_name; } + // Cooperative matrix -> Metal simdgroup matrix type + { + const SPIRType *coop_type = &type; + while (coop_type && (is_pointer(*coop_type) || is_array(*coop_type))) + coop_type = maybe_get(coop_type->parent_type); + + if (coop_type && coop_type->op == OpTypeCooperativeMatrixKHR) + { + if (!msl_options.supports_msl_version(3, 1)) + SPIRV_CROSS_THROW("Cooperative matrices require MSL 3.1 or later."); + + // Only Subgroup scope + auto &scope_c = get(coop_type->ext.cooperative.scope_id); + if (scope_c.specialization) + SPIRV_CROSS_THROW("MSL does not support spec-constant scope for cooperative matrices."); + if (scope_c.scalar() != ScopeSubgroup) + SPIRV_CROSS_THROW("MSL cooperative matrices only support Subgroup scope."); + + // Only 8x8 + auto &rows_c = get(coop_type->ext.cooperative.rows_id); + auto &cols_c = get(coop_type->ext.cooperative.columns_id); + if (rows_c.specialization || cols_c.specialization) + SPIRV_CROSS_THROW("MSL does not support spec-constant dimensions for cooperative matrices."); + if (rows_c.scalar() != 8 || cols_c.scalar() != 8) + SPIRV_CROSS_THROW("MSL cooperative matrices only support 8x8 dimensions."); + + // Map component type to simdgroup_*8x8 + auto &comp = get(coop_type->parent_type); + switch (comp.basetype) + { + case SPIRType::Float: + return "simdgroup_float8x8"; + case SPIRType::Half: + return "simdgroup_half8x8"; + case SPIRType::BFloat16: + return "simdgroup_bfloat8x8"; + default: + SPIRV_CROSS_THROW("Unsupported component type for MSL cooperative matrix."); + } + } + } + switch (type.basetype) { case SPIRType::Struct: @@ -15107,10 +17082,10 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) case SPIRType::Image: case SPIRType::SampledImage: - return image_type_glsl(type, id); + return image_type_glsl(type, id, member); case SPIRType::Sampler: - return sampler_type(type, id); + return sampler_type(type, id, member); case SPIRType::Void: return "void"; @@ -15180,6 +17155,11 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) case SPIRType::Double: type_name = "double"; // Currently unsupported break; + case SPIRType::BFloat16: + if (!msl_options.supports_msl_version(3, 1)) + SPIRV_CROSS_THROW("bfloat16 requires MSL 3.1 or later."); + type_name = "bfloat"; + break; case SPIRType::AccelerationStructure: if (msl_options.supports_msl_version(2, 4)) type_name = "raytracing::acceleration_structure"; @@ -15190,6 +17170,8 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id, bool member) break; case SPIRType::RayQuery: return "raytracing::intersection_query"; + case SPIRType::MeshGridProperties: + return "mesh_grid_properties"; default: return "unknown_type"; @@ -15249,7 +17231,7 @@ string CompilerMSL::type_to_glsl(const SPIRType &type, uint32_t id) return type_to_glsl(type, id, false); } -string CompilerMSL::type_to_array_glsl(const SPIRType &type) +string CompilerMSL::type_to_array_glsl(const SPIRType &type, uint32_t variable_id) { // Allow Metal to use the array template to make arrays a value type switch (type.basetype) @@ -15257,11 +17239,20 @@ string CompilerMSL::type_to_array_glsl(const SPIRType &type) case SPIRType::AtomicCounter: case SPIRType::ControlPointArray: case SPIRType::RayQuery: - return CompilerGLSL::type_to_array_glsl(type); + return CompilerGLSL::type_to_array_glsl(type, variable_id); default: if (type_is_array_of_pointers(type) || using_builtin_array()) - return CompilerGLSL::type_to_array_glsl(type); + { + const SPIRVariable *var = variable_id ? maybe_get(variable_id) : nullptr; + if (var && (var->storage == StorageClassUniform || var->storage == StorageClassStorageBuffer) && + is_array(get_variable_data_type(*var))) + { + return join("[", get_resource_array_size(type, variable_id), "]"); + } + else + return CompilerGLSL::type_to_array_glsl(type, variable_id); + } else return ""; } @@ -15271,6 +17262,9 @@ string CompilerMSL::constant_op_expression(const SPIRConstantOp &cop) { switch (cop.opcode) { + case OpSMod: + add_spv_func_and_recompile(SPVFuncImplSMod); + return join("spvSMod(", to_expression(cop.arguments[0]), ", ", to_expression(cop.arguments[1]), ")"); case OpQuantizeToF16: add_spv_func_and_recompile(SPVFuncImplQuantizeToF16); return join("spvQuantizeToF16(", to_expression(cop.arguments[0]), ")"); @@ -15279,7 +17273,7 @@ string CompilerMSL::constant_op_expression(const SPIRConstantOp &cop) } } -bool CompilerMSL::variable_decl_is_remapped_storage(const SPIRVariable &variable, spv::StorageClass storage) const +bool CompilerMSL::variable_decl_is_remapped_storage(const SPIRVariable &variable, StorageClass storage) const { if (variable.storage == storage) return true; @@ -15295,6 +17289,9 @@ bool CompilerMSL::variable_decl_is_remapped_storage(const SPIRVariable &variable return true; } + if (is_mesh_shader()) + return variable.storage == StorageClassOutput; + return variable.storage == StorageClassOutput && is_tesc_shader() && is_stage_output_variable_masked(variable); } else if (storage == StorageClassStorageBuffer) @@ -15326,7 +17323,7 @@ std::string CompilerMSL::variable_decl(const SPIRType &type, const std::string & return CompilerGLSL::variable_decl(type, name, id); } -std::string CompilerMSL::sampler_type(const SPIRType &type, uint32_t id) +std::string CompilerMSL::sampler_type(const SPIRType &type, uint32_t id, bool member) { auto *var = maybe_get(id); if (var && var->basevariable) @@ -15345,26 +17342,31 @@ std::string CompilerMSL::sampler_type(const SPIRType &type, uint32_t id) // Arrays of samplers in MSL must be declared with a special array syntax ala C++11 std::array. // If we have a runtime array, it could be a variable-count descriptor set binding. + auto &parent = get(get_pointee_type(type).parent_type); uint32_t array_size = get_resource_array_size(type, id); + if (array_size == 0) { add_spv_func_and_recompile(SPVFuncImplVariableDescriptor); add_spv_func_and_recompile(SPVFuncImplVariableDescriptorArray); - auto &parent = get(get_pointee_type(type).parent_type); - if (processing_entry_point) - return join("const device spvDescriptor<", sampler_type(parent, id), ">*"); - return join("const spvDescriptorArray<", sampler_type(parent, id), ">"); - } - auto &parent = get(get_pointee_type(type).parent_type); - return join("array<", sampler_type(parent, id), ", ", array_size, ">"); + const char *descriptor_wrapper = processing_entry_point ? "const device spvDescriptor" : "const spvDescriptorArray"; + if (member) + descriptor_wrapper = "spvDescriptor"; + return join(descriptor_wrapper, "<", sampler_type(parent, id, false), ">", + processing_entry_point ? "*" : ""); + } + else + { + return join("array<", sampler_type(parent, id, false), ", ", array_size, ">"); + } } else return "sampler"; } // Returns an MSL string describing the SPIR-V image type -string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id) +string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id, bool member) { auto *var = maybe_get(id); if (var && var->basevariable) @@ -15395,23 +17397,35 @@ string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id) // Arrays of images in MSL must be declared with a special array syntax ala C++11 std::array. // If we have a runtime array, it could be a variable-count descriptor set binding. + auto &parent = get(get_pointee_type(type).parent_type); uint32_t array_size = get_resource_array_size(type, id); + if (array_size == 0) { add_spv_func_and_recompile(SPVFuncImplVariableDescriptor); add_spv_func_and_recompile(SPVFuncImplVariableDescriptorArray); - auto &parent = get(get_pointee_type(type).parent_type); - return join("const device spvDescriptor<", image_type_glsl(parent, id), ">*"); + const char *descriptor_wrapper = processing_entry_point ? "const device spvDescriptor" : "const spvDescriptorArray"; + if (member) + { + descriptor_wrapper = "spvDescriptor"; + // This requires a specialized wrapper type that packs image and sampler side by side. + // It is possible in theory. + if (type.basetype == SPIRType::SampledImage) + SPIRV_CROSS_THROW("Argument buffer runtime array currently not supported for combined image sampler."); + } + return join(descriptor_wrapper, "<", image_type_glsl(parent, id, false), ">", + processing_entry_point ? "*" : ""); + } + else + { + return join("array<", image_type_glsl(parent, id, false), ", ", array_size, ">"); } - - auto &parent = get(get_pointee_type(type).parent_type); - return join("array<", image_type_glsl(parent, id), ", ", array_size, ">"); } string img_type_name; - // Bypass pointers because we need the real image struct - auto &img_type = get(type.self).image; + auto &img_type = type.image; + if (is_depth_image(type, id)) { switch (img_type.dim) @@ -15527,6 +17541,12 @@ string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id) // Otherwise it may be set based on whether the image is read from or written to within the shader. if (type.basetype == SPIRType::Image && type.image.sampled == 2 && type.image.dim != DimSubpassData) { + auto *p_var = maybe_get_backing_variable(id); + if (p_var && p_var->basevariable) + p_var = maybe_get(p_var->basevariable); + + bool has_access_qualifier = true; + switch (img_type.access) { case AccessQualifierReadOnly: @@ -15543,9 +17563,6 @@ string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id) default: { - auto *p_var = maybe_get_backing_variable(id); - if (p_var && p_var->basevariable) - p_var = maybe_get(p_var->basevariable); if (p_var && !has_decoration(p_var->self, DecorationNonWritable)) { img_type_name += ", access::"; @@ -15555,9 +17572,21 @@ string CompilerMSL::image_type_glsl(const SPIRType &type, uint32_t id) img_type_name += "write"; } + else + { + has_access_qualifier = false; + } break; } } + + if (p_var && has_decoration(p_var->self, DecorationCoherent) && msl_options.supports_msl_version(3, 2)) + { + // Cannot declare memory_coherence_device without access qualifier. + if (!has_access_qualifier) + img_type_name += ", access::read"; + img_type_name += ", memory_coherence_device"; + } } img_type_name += ">"; @@ -15625,6 +17654,10 @@ void CompilerMSL::emit_subgroup_op(const Instruction &i) if (!msl_options.supports_msl_version(2, 2)) SPIRV_CROSS_THROW("Ballot ops on iOS requires Metal 2.2 and up."); break; + case OpGroupNonUniformRotateKHR: + if (!msl_options.supports_msl_version(2, 2)) + SPIRV_CROSS_THROW("Rotate on iOS requires Metal 2.2 and up."); + break; case OpGroupNonUniformBroadcast: case OpGroupNonUniformShuffle: case OpGroupNonUniformShuffleXor: @@ -15660,13 +17693,16 @@ void CompilerMSL::emit_subgroup_op(const Instruction &i) Scope scope; switch (op) { + // These earlier instructions don't have the scope operand. case OpSubgroupBallotKHR: case OpSubgroupFirstInvocationKHR: case OpSubgroupReadInvocationKHR: case OpSubgroupAllKHR: case OpSubgroupAnyKHR: case OpSubgroupAllEqualKHR: - // These earlier instructions don't have the scope operand. + // These instructions are always quad-scoped and thus do not have a scope operand. + case OpGroupNonUniformQuadAllKHR: + case OpGroupNonUniformQuadAnyKHR: scope = ScopeSubgroup; break; default: @@ -15754,6 +17790,23 @@ void CompilerMSL::emit_subgroup_op(const Instruction &i) emit_binary_func_op(result_type, id, ops[op_idx], ops[op_idx + 1], "spvSubgroupShuffleDown"); break; + case OpGroupNonUniformRotateKHR: + { + if (i.length > 5) + { + // MSL does not have a cluster size parameter, so calculate the invocation ID manually and using a shuffle. + auto delta_expr = enclose_expression(to_unpacked_expression(ops[op_idx + 1])); + auto cluster_size_minus_one = evaluate_constant_u32(ops[op_idx + 2]) - 1; + auto local_id_expr = to_unpacked_expression(scope == ScopeSubgroup + ? builtin_subgroup_invocation_id_id : builtin_local_invocation_index_id); + auto shuffle_idx = join("((", local_id_expr, " + ", delta_expr, ")", " & ", std::to_string(cluster_size_minus_one), + ") + (", local_id_expr, " & ", std::to_string(~cluster_size_minus_one), ")"); + emit_op(result_type, id, join("spvSubgroupShuffle(", to_unpacked_expression(ops[op_idx]), ", ", shuffle_idx, ")"), false); + } else + emit_binary_func_op(result_type, id, ops[op_idx], ops[op_idx + 1], "spvSubgroupRotate"); + break; + } + case OpGroupNonUniformAll: case OpSubgroupAllKHR: if (msl_options.use_quadgroup_operation()) @@ -15788,11 +17841,10 @@ case OpGroupNonUniform##op: \ emit_unary_func_op(result_type, id, ops[op_idx], "simd_prefix_exclusive_" #msl_op); \ else if (operation == GroupOperationClusteredReduce) \ { \ - /* Only cluster sizes of 4 are supported. */ \ uint32_t cluster_size = evaluate_constant_u32(ops[op_idx + 1]); \ - if (cluster_size != 4) \ - SPIRV_CROSS_THROW("Metal only supports quad ClusteredReduce."); \ - emit_unary_func_op(result_type, id, ops[op_idx], "quad_" #msl_op); \ + if (get_execution_model() != ExecutionModelFragment || msl_options.supports_msl_version(2, 2)) \ + add_spv_func_and_recompile(SPVFuncImplSubgroupClustered##op); \ + emit_subgroup_cluster_op(result_type, id, cluster_size, ops[op_idx], #msl_op); \ } \ else \ SPIRV_CROSS_THROW("Invalid group operation."); \ @@ -15817,11 +17869,10 @@ case OpGroupNonUniform##op: \ SPIRV_CROSS_THROW("Metal doesn't support ExclusiveScan for OpGroupNonUniform" #op "."); \ else if (operation == GroupOperationClusteredReduce) \ { \ - /* Only cluster sizes of 4 are supported. */ \ uint32_t cluster_size = evaluate_constant_u32(ops[op_idx + 1]); \ - if (cluster_size != 4) \ - SPIRV_CROSS_THROW("Metal only supports quad ClusteredReduce."); \ - emit_unary_func_op(result_type, id, ops[op_idx], "quad_" #msl_op); \ + if (get_execution_model() != ExecutionModelFragment || msl_options.supports_msl_version(2, 2)) \ + add_spv_func_and_recompile(SPVFuncImplSubgroupClustered##op); \ + emit_subgroup_cluster_op(result_type, id, cluster_size, ops[op_idx], #msl_op); \ } \ else \ SPIRV_CROSS_THROW("Invalid group operation."); \ @@ -15840,11 +17891,10 @@ case OpGroupNonUniform##op: \ SPIRV_CROSS_THROW("Metal doesn't support ExclusiveScan for OpGroupNonUniform" #op "."); \ else if (operation == GroupOperationClusteredReduce) \ { \ - /* Only cluster sizes of 4 are supported. */ \ uint32_t cluster_size = evaluate_constant_u32(ops[op_idx + 1]); \ - if (cluster_size != 4) \ - SPIRV_CROSS_THROW("Metal only supports quad ClusteredReduce."); \ - emit_unary_func_op_cast(result_type, id, ops[op_idx], "quad_" #msl_op, type, type); \ + if (get_execution_model() != ExecutionModelFragment || msl_options.supports_msl_version(2, 2)) \ + add_spv_func_and_recompile(SPVFuncImplSubgroupClustered##op); \ + emit_subgroup_cluster_op_cast(result_type, id, cluster_size, ops[op_idx], #msl_op, type, type); \ } \ else \ SPIRV_CROSS_THROW("Invalid group operation."); \ @@ -15860,9 +17910,11 @@ case OpGroupNonUniform##op: \ MSL_GROUP_OP(BitwiseAnd, and) MSL_GROUP_OP(BitwiseOr, or) MSL_GROUP_OP(BitwiseXor, xor) - MSL_GROUP_OP(LogicalAnd, and) - MSL_GROUP_OP(LogicalOr, or) - MSL_GROUP_OP(LogicalXor, xor) + // Metal doesn't support boolean types in SIMD-group operations, so we + // have to emit some casts. + MSL_GROUP_OP_CAST(LogicalAnd, and, SPIRType::UShort) + MSL_GROUP_OP_CAST(LogicalOr, or, SPIRType::UShort) + MSL_GROUP_OP_CAST(LogicalXor, xor, SPIRType::UShort) // clang-format on #undef MSL_GROUP_OP #undef MSL_GROUP_OP_CAST @@ -15875,6 +17927,14 @@ case OpGroupNonUniform##op: \ emit_binary_func_op(result_type, id, ops[op_idx], ops[op_idx + 1], "spvQuadBroadcast"); break; + case OpGroupNonUniformQuadAllKHR: + emit_unary_func_op(result_type, id, ops[op_idx], "quad_all"); + break; + + case OpGroupNonUniformQuadAnyKHR: + emit_unary_func_op(result_type, id, ops[op_idx], "quad_any"); + break; + default: SPIRV_CROSS_THROW("Invalid opcode for subgroup."); } @@ -15882,13 +17942,95 @@ case OpGroupNonUniform##op: \ register_control_dependent_expression(id); } +void CompilerMSL::emit_subgroup_cluster_op(uint32_t result_type, uint32_t result_id, uint32_t cluster_size, + uint32_t op0, const char *op) +{ + if (get_execution_model() == ExecutionModelFragment && !msl_options.supports_msl_version(2, 2)) + { + if (cluster_size == 4) + { + emit_unary_func_op(result_type, result_id, op0, join("quad_", op).c_str()); + return; + } + SPIRV_CROSS_THROW("Cluster sizes other than 4 in fragment shaders require MSL 2.2."); + } + bool forward = should_forward(op0); + emit_op(result_type, result_id, + join("spvClustered_", op, "<", cluster_size, ">(", to_unpacked_expression(op0), ", ", + to_expression(builtin_subgroup_invocation_id_id), ")"), + forward); + inherit_expression_dependencies(result_id, op0); +} + +void CompilerMSL::emit_subgroup_cluster_op_cast(uint32_t result_type, uint32_t result_id, uint32_t cluster_size, + uint32_t op0, const char *op, SPIRType::BaseType input_type, + SPIRType::BaseType expected_result_type) +{ + if (get_execution_model() == ExecutionModelFragment && !msl_options.supports_msl_version(2, 2)) + { + if (cluster_size == 4) + { + emit_unary_func_op_cast(result_type, result_id, op0, join("quad_", op).c_str(), input_type, + expected_result_type); + return; + } + SPIRV_CROSS_THROW("Cluster sizes other than 4 in fragment shaders require MSL 2.2."); + } + + auto &out_type = get(result_type); + auto &expr_type = expression_type(op0); + auto expected_type = out_type; + + // Bit-widths might be different in unary cases because we use it for SConvert/UConvert and friends. + expected_type.basetype = input_type; + expected_type.width = expr_type.width; + + string cast_op; + if (expr_type.basetype != input_type) + { + if (expr_type.basetype == SPIRType::Boolean) + cast_op = join(type_to_glsl(expected_type), "(", to_unpacked_expression(op0), ")"); + else + cast_op = bitcast_glsl(expected_type, op0); + } + else + cast_op = to_unpacked_expression(op0); + + string sg_op = join("spvClustered_", op, "<", cluster_size, ">"); + string expr; + if (out_type.basetype != expected_result_type) + { + expected_type.basetype = expected_result_type; + expected_type.width = out_type.width; + if (out_type.basetype == SPIRType::Boolean) + expr = type_to_glsl(out_type); + else + expr = bitcast_glsl_op(out_type, expected_type); + expr += '('; + expr += join(sg_op, "(", cast_op, ", ", to_expression(builtin_subgroup_invocation_id_id), ")"); + expr += ')'; + } + else + { + expr += join(sg_op, "(", cast_op, ", ", to_expression(builtin_subgroup_invocation_id_id), ")"); + } + + emit_op(result_type, result_id, expr, should_forward(op0)); + inherit_expression_dependencies(result_id, op0); +} + +// Note: Metal forbids bitcasting to/from 'bool' using as_type. This function is used widely +// for generating casts in the backend. To avoid generating illegal MSL when the canonical +// function constant type (from deduplicated SpecId) is Boolean, fall back to value-cast in +// that case by returning type_to_glsl(out_type) instead of as_type<...>. string CompilerMSL::bitcast_glsl_op(const SPIRType &out_type, const SPIRType &in_type) { if (out_type.basetype == in_type.basetype) return ""; - assert(out_type.basetype != SPIRType::Boolean); - assert(in_type.basetype != SPIRType::Boolean); + // Avoid bitcasting to/from booleans in MSL; use value cast instead. + if (out_type.basetype == SPIRType::Boolean || in_type.basetype == SPIRType::Boolean) + return type_to_glsl(out_type); bool integral_cast = type_is_integral(out_type) && type_is_integral(in_type) && (out_type.vecsize == in_type.vecsize); bool same_size_cast = (out_type.width * out_type.vecsize) == (in_type.width * in_type.vecsize); @@ -16024,8 +18166,9 @@ string CompilerMSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) { SPIRV_CROSS_THROW("BaseInstance requires Metal 1.1 and Mac or Apple A9+ hardware."); } + case BuiltInDrawIndex: - SPIRV_CROSS_THROW("DrawIndex is not supported in MSL."); + return "gl_DrawID"; // When used in the entry function, output builtins are qualified with output struct name. // Test storage class as NOT Input, as output builtins might be part of generic type. @@ -16047,6 +18190,8 @@ string CompilerMSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) case BuiltInLayer: if (is_tesc_shader()) break; + if (is_mesh_shader()) + break; if (storage != StorageClassInput && current_function && (current_function->self == ir.default_entry_point) && !is_stage_output_builtin_masked(builtin)) return stage_out_var_name + "." + CompilerGLSL::builtin_to_glsl(builtin, storage); @@ -16104,6 +18249,9 @@ string CompilerMSL::builtin_to_glsl(BuiltIn builtin, StorageClass storage) // In SPIR-V 1.6 with Volatile HelperInvocation, we cannot emit a fixup early. return "simd_is_helper_thread()"; + case BuiltInPrimitiveId: + return "gl_PrimitiveID"; + default: break; } @@ -16131,12 +18279,12 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) return "instance_id"; case BuiltInBaseInstance: return "base_instance"; - case BuiltInDrawIndex: - SPIRV_CROSS_THROW("DrawIndex is not supported in MSL."); // Vertex function out case BuiltInClipDistance: return "clip_distance"; + case BuiltInCullDistance: + return "cull_distance"; case BuiltInPointSize: return "point_size"; case BuiltInPosition: @@ -16184,6 +18332,8 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) else if (msl_options.is_macos() && !msl_options.supports_msl_version(2, 2)) SPIRV_CROSS_THROW("PrimitiveId on macOS requires MSL 2.2."); return "primitive_id"; + case ExecutionModelMeshEXT: + return "primitive_id"; default: SPIRV_CROSS_THROW("PrimitiveId is not supported in this execution model."); } @@ -16213,7 +18363,7 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) // Shouldn't be reached. SPIRV_CROSS_THROW("Sample position is retrieved by a function in MSL."); case BuiltInViewIndex: - if (execution.model != ExecutionModelFragment) + if (execution.model != ExecutionModelFragment && execution.model != ExecutionModelMeshEXT) SPIRV_CROSS_THROW("ViewIndex is handled specially outside fragment shaders."); // The ViewIndex was implicitly used in the prior stages to set the render_target_array_index, // so we can get it from there. @@ -16235,6 +18385,9 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) case BuiltInGlobalInvocationId: return "thread_position_in_grid"; + case BuiltInWorkgroupSize: + return "threads_per_threadgroup"; + case BuiltInWorkgroupId: return "threadgroup_position_in_grid"; @@ -16261,7 +18414,10 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) { // thread_execution_width is an alias for threads_per_simdgroup, and it's only available since 1.0, // but not in fragment. - return "thread_execution_width"; + if (msl_options.supports_msl_version(3, 0)) + return "threads_per_simdgroup"; + else + return "thread_execution_width"; } case BuiltInNumSubgroups: @@ -16291,6 +18447,7 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) return "thread_index_in_simdgroup"; } else if (execution.model == ExecutionModelKernel || execution.model == ExecutionModelGLCompute || + execution.model == ExecutionModelTaskEXT || execution.model == ExecutionModelMeshEXT || execution.model == ExecutionModelTessellationControl || (execution.model == ExecutionModelVertex && msl_options.vertex_for_tessellation)) { @@ -16311,18 +18468,15 @@ string CompilerMSL::builtin_qualifier(BuiltIn builtin) SPIRV_CROSS_THROW("Subgroup ballot masks are handled specially in MSL."); case BuiltInBaryCoordKHR: - if (msl_options.is_ios() && !msl_options.supports_msl_version(2, 3)) - SPIRV_CROSS_THROW("Barycentrics are only supported in MSL 2.3 and above on iOS."); - else if (!msl_options.supports_msl_version(2, 2)) - SPIRV_CROSS_THROW("Barycentrics are only supported in MSL 2.2 and above on macOS."); - return "barycentric_coord, center_perspective"; - case BuiltInBaryCoordNoPerspKHR: if (msl_options.is_ios() && !msl_options.supports_msl_version(2, 3)) SPIRV_CROSS_THROW("Barycentrics are only supported in MSL 2.3 and above on iOS."); else if (!msl_options.supports_msl_version(2, 2)) SPIRV_CROSS_THROW("Barycentrics are only supported in MSL 2.2 and above on macOS."); - return "barycentric_coord, center_no_perspective"; + return "barycentric_coord"; + + case BuiltInCullPrimitiveEXT: + return "primitive_culled"; default: return "unsupported-built-in"; @@ -16348,7 +18502,7 @@ string CompilerMSL::builtin_type_decl(BuiltIn builtin, uint32_t id) case BuiltInBaseInstance: return "uint"; case BuiltInDrawIndex: - SPIRV_CROSS_THROW("DrawIndex is not supported in MSL."); + return "uint"; // Vertex function out case BuiltInClipDistance: @@ -16423,6 +18577,7 @@ string CompilerMSL::builtin_type_decl(BuiltIn builtin, uint32_t id) case BuiltInLocalInvocationId: case BuiltInNumWorkgroups: case BuiltInWorkgroupId: + case BuiltInWorkgroupSize: return "uint3"; case BuiltInLocalInvocationIndex: case BuiltInNumSubgroups: @@ -16440,6 +18595,13 @@ string CompilerMSL::builtin_type_decl(BuiltIn builtin, uint32_t id) case BuiltInDeviceIndex: return "int"; + case BuiltInPrimitivePointIndicesEXT: + return "uint"; + case BuiltInPrimitiveLineIndicesEXT: + return "uint2"; + case BuiltInPrimitiveTriangleIndicesEXT: + return "uint3"; + default: return "unsupported-built-in-type"; } @@ -16462,17 +18624,17 @@ string CompilerMSL::built_in_func_arg(BuiltIn builtin, bool prefix_comma) return bi_arg; } -const SPIRType &CompilerMSL::get_physical_member_type(const SPIRType &type, uint32_t index) const +TypeID CompilerMSL::get_physical_member_type_id(const SPIRType &type, uint32_t index) const { if (member_is_remapped_physical_type(type, index)) - return get(get_extended_member_decoration(type.self, index, SPIRVCrossDecorationPhysicalTypeID)); + return get_extended_member_decoration(type.self, index, SPIRVCrossDecorationPhysicalTypeID); else - return get(type.member_types[index]); + return type.member_types[index]; } SPIRType CompilerMSL::get_presumed_input_type(const SPIRType &ib_type, uint32_t index) const { - SPIRType type = get_physical_member_type(ib_type, index); + SPIRType type = get(get_physical_member_type_id(ib_type, index)); uint32_t loc = get_member_decoration(ib_type.self, index, DecorationLocation); uint32_t cmp = get_member_decoration(ib_type.self, index, DecorationComponent); auto p_va = inputs_by_location.find({loc, cmp}); @@ -16482,7 +18644,7 @@ SPIRType CompilerMSL::get_presumed_input_type(const SPIRType &ib_type, uint32_t return type; } -uint32_t CompilerMSL::get_declared_type_array_stride_msl(const SPIRType &type, bool is_packed, bool row_major) const +uint32_t CompilerMSL::get_declared_type_array_stride_msl(TypeID type_id, const SPIRType *special_type, bool is_packed, bool row_major) const { // Array stride in MSL is always size * array_size. sizeof(float3) == 16, // unlike GLSL and HLSL where array stride would be 16 and size 12. @@ -16491,11 +18653,42 @@ uint32_t CompilerMSL::get_declared_type_array_stride_msl(const SPIRType &type, b // far more complicated. We'd rather just create the final type, and ignore having to create the entire type // hierarchy in order to compute this value, so make a temporary type on the stack. - auto basic_type = type; - basic_type.array.clear(); - basic_type.array_size_literal.clear(); - uint32_t value_size = get_declared_type_size_msl(basic_type, is_packed, row_major); + uint32_t value_size; + // We don't always use proper type hierarchy for synthesized types, so be robust. + if (type_id && get(type_id).parent_type) + { + bool uses_declared_array_stride = false; + + uint32_t array_stride = 0; + TypeID basic_type_id = type_id; + while (is_array(get(basic_type_id))) + { + array_stride = get_decoration(basic_type_id, DecorationArrayStride); + auto parent_type_id = get(basic_type_id).parent_type; + // If the base struct itself has ArrayStride decoration, it will be padded on-demand. + uses_declared_array_stride = has_decoration(parent_type_id, DecorationArrayStride); + if (parent_type_id) + basic_type_id = parent_type_id; + else + break; + } + + if (array_stride && uses_declared_array_stride) + value_size = array_stride; + else + value_size = get_declared_type_size_msl(basic_type_id, nullptr, is_packed, row_major); + } + else + { + // Old, broken path. + auto basic_type = type_id ? get(type_id) : *special_type; + basic_type.array.clear(); + basic_type.array_size_literal.clear(); + value_size = get_declared_type_size_msl(0, &basic_type, is_packed, row_major); + } + + auto &type = type_id ? get(type_id) : *special_type; uint32_t dimensions = uint32_t(type.array.size()); assert(dimensions > 0); dimensions--; @@ -16512,47 +18705,47 @@ uint32_t CompilerMSL::get_declared_type_array_stride_msl(const SPIRType &type, b uint32_t CompilerMSL::get_declared_struct_member_array_stride_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_array_stride_msl(get_physical_member_type(type, index), + return get_declared_type_array_stride_msl(get_physical_member_type_id(type, index), nullptr, member_is_packed_physical_type(type, index), has_member_decoration(type.self, index, DecorationRowMajor)); } uint32_t CompilerMSL::get_declared_input_array_stride_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_array_stride_msl(get_presumed_input_type(type, index), false, + auto presumed_type = get_presumed_input_type(type, index); + return get_declared_type_array_stride_msl(0, &presumed_type, false, has_member_decoration(type.self, index, DecorationRowMajor)); } -uint32_t CompilerMSL::get_declared_type_matrix_stride_msl(const SPIRType &type, bool packed, bool row_major) const +uint32_t CompilerMSL::get_declared_type_matrix_stride_msl(TypeID type_id, const SPIRType *special_type, + bool packed, bool row_major) const { + auto &type = type_id ? get(type_id) : *special_type; + // For packed matrices, we just use the size of the vector type. // Otherwise, MatrixStride == alignment, which is the size of the underlying vector type. if (packed) return (type.width / 8) * ((row_major && type.columns > 1) ? type.columns : type.vecsize); else - return get_declared_type_alignment_msl(type, false, row_major); + return get_declared_type_alignment_msl(type_id, special_type, false, row_major); } uint32_t CompilerMSL::get_declared_struct_member_matrix_stride_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_matrix_stride_msl(get_physical_member_type(type, index), + return get_declared_type_matrix_stride_msl(get_physical_member_type_id(type, index), nullptr, member_is_packed_physical_type(type, index), has_member_decoration(type.self, index, DecorationRowMajor)); } uint32_t CompilerMSL::get_declared_input_matrix_stride_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_matrix_stride_msl(get_presumed_input_type(type, index), false, + auto presumed_type = get_presumed_input_type(type, index); + return get_declared_type_matrix_stride_msl(0, &presumed_type, false, has_member_decoration(type.self, index, DecorationRowMajor)); } -uint32_t CompilerMSL::get_declared_struct_size_msl(const SPIRType &struct_type, bool ignore_alignment, - bool ignore_padding) const +uint32_t CompilerMSL::get_declared_struct_size_msl(const SPIRType &struct_type) const { - // If we have a target size, that is the declared size as well. - if (!ignore_padding && has_extended_decoration(struct_type.self, SPIRVCrossDecorationPaddingTarget)) - return get_extended_decoration(struct_type.self, SPIRVCrossDecorationPaddingTarget); - if (struct_type.member_types.empty()) return 0; @@ -16561,13 +18754,10 @@ uint32_t CompilerMSL::get_declared_struct_size_msl(const SPIRType &struct_type, // In MSL, a struct's alignment is equal to the maximum alignment of any of its members. uint32_t alignment = 1; - if (!ignore_alignment) + for (uint32_t i = 0; i < mbr_cnt; i++) { - for (uint32_t i = 0; i < mbr_cnt; i++) - { - uint32_t mbr_alignment = get_declared_struct_member_alignment_msl(struct_type, i); - alignment = max(alignment, mbr_alignment); - } + uint32_t mbr_alignment = get_declared_struct_member_alignment_msl(struct_type, i); + alignment = max(alignment, mbr_alignment); } // Last member will always be matched to the final Offset decoration, but size of struct in MSL now depends @@ -16578,19 +18768,30 @@ uint32_t CompilerMSL::get_declared_struct_size_msl(const SPIRType &struct_type, return msl_size; } -// Returns the byte size of a struct member. -uint32_t CompilerMSL::get_declared_type_size_msl(const SPIRType &type, bool is_packed, bool row_major) const +uint32_t CompilerMSL::get_physical_type_id_stride(TypeID type_id) const { + // This should only be relevant for plain types such as scalars and vectors? + // If we're pointing to a struct, it will recursively pick up packed/row-major state. + return get_declared_type_size_msl(type_id, nullptr, false, false); +} + +// Returns the byte size of a struct member. +uint32_t CompilerMSL::get_declared_type_size_msl(TypeID type_id, const SPIRType *special_type, + bool is_packed, bool row_major) const +{ + auto &type = type_id ? get(type_id) : *special_type; + // Pointers take 8 bytes each + // Match both pointer and array-of-pointer here. if (type.pointer && type.storage == StorageClassPhysicalStorageBuffer) { - uint32_t type_size = 8 * (type.vecsize == 3 ? 4 : type.vecsize); + uint32_t type_size = 8; // Work our way through potentially layered arrays, // stopping when we hit a pointer that is not also an array. int32_t dim_idx = (int32_t)type.array.size() - 1; auto *p_type = &type; - while (!type_is_pointer(*p_type) && dim_idx >= 0) + while (!is_pointer(*p_type) && dim_idx >= 0) { type_size *= to_array_size_literal(*p_type, dim_idx); p_type = &get(p_type->parent_type); @@ -16612,10 +18813,27 @@ uint32_t CompilerMSL::get_declared_type_size_msl(const SPIRType &type, bool is_p default: { - if (!type.array.empty()) + if ((!type.parent_type || special_type) && !type.array.empty()) { + // Special case where the type hierarchy is not set up properly. + // Don't want to have to allocate a bunch of dummy type IDs just to make it work. uint32_t array_size = to_array_size_literal(type); - return get_declared_type_array_stride_msl(type, is_packed, row_major) * max(array_size, 1u); + return get_declared_type_array_stride_msl(type_id, special_type, is_packed, row_major) * max(array_size, 1u); + } + else if (is_array(type) && type.parent_type) + { + // For the proper case. Ideally all code paths should go through here, but + // would need a lot of cleanup to make that work ... + auto &parent_type = get(type.parent_type); + uint32_t effective_stride; + + if (parent_type.op == OpTypeStruct && has_decoration(parent_type.self, DecorationArrayStride)) + effective_stride = get_decoration(type_id, DecorationArrayStride); + else + effective_stride = get_declared_type_array_stride_msl(type_id, special_type, is_packed, row_major); + + uint32_t array_size = to_array_size_literal(type); + return effective_stride * max(array_size, 1u); } if (type.basetype == SPIRType::Struct) @@ -16645,23 +18863,28 @@ uint32_t CompilerMSL::get_declared_type_size_msl(const SPIRType &type, bool is_p uint32_t CompilerMSL::get_declared_struct_member_size_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_size_msl(get_physical_member_type(type, index), + return get_declared_type_size_msl(get_physical_member_type_id(type, index), nullptr, member_is_packed_physical_type(type, index), has_member_decoration(type.self, index, DecorationRowMajor)); } uint32_t CompilerMSL::get_declared_input_size_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_size_msl(get_presumed_input_type(type, index), false, + auto presumed_type = get_presumed_input_type(type, index); + return get_declared_type_size_msl(0, &presumed_type, false, has_member_decoration(type.self, index, DecorationRowMajor)); } // Returns the byte alignment of a type. -uint32_t CompilerMSL::get_declared_type_alignment_msl(const SPIRType &type, bool is_packed, bool row_major) const +uint32_t CompilerMSL::get_declared_type_alignment_msl(TypeID type_id, const SPIRType *special_type, + bool is_packed, bool row_major) const { - // Pointers aligns on multiples of 8 bytes + auto &type = type_id ? get(type_id) : *special_type; + + // Pointers align on multiples of 8 bytes. + // Deliberately ignore array-ness here. It's not relevant for alignment. if (type.pointer && type.storage == StorageClassPhysicalStorageBuffer) - return 8 * (type.vecsize == 3 ? 4 : type.vecsize); + return 8; switch (type.basetype) { @@ -16711,14 +18934,15 @@ uint32_t CompilerMSL::get_declared_type_alignment_msl(const SPIRType &type, bool uint32_t CompilerMSL::get_declared_struct_member_alignment_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_alignment_msl(get_physical_member_type(type, index), + return get_declared_type_alignment_msl(get_physical_member_type_id(type, index), nullptr, member_is_packed_physical_type(type, index), has_member_decoration(type.self, index, DecorationRowMajor)); } uint32_t CompilerMSL::get_declared_input_alignment_msl(const SPIRType &type, uint32_t index) const { - return get_declared_type_alignment_msl(get_presumed_input_type(type, index), false, + auto presumed_type = get_presumed_input_type(type, index); + return get_declared_type_alignment_msl(0, &presumed_type, false, has_member_decoration(type.self, index, DecorationRowMajor)); } @@ -16736,7 +18960,24 @@ void CompilerMSL::analyze_sampled_image_usage() } } -bool CompilerMSL::SampledImageScanner::handle(spv::Op opcode, const uint32_t *args, uint32_t length) +void CompilerMSL::analyze_workgroup_variables() +{ + ir.for_each_typed_id([&](uint32_t, SPIRVariable &var) { + // If workgroup variables have initializer, it can only be ConstantNull (zero init) + if (var.storage == StorageClassWorkgroup && var.initializer) + { + needs_workgroup_zero_init = true; + + // MSL compiler does not like the routine to initialize simple threadgroup variables, + // falsely claiming it is "sometimes uninitialized". Suppress it. + auto &type = get_variable_data_type(var); + if (type.array.empty() && type.member_types.empty()) + suppress_sometimes_unitialized = true; + } + }); +} + +bool CompilerMSL::SampledImageScanner::handle(Op opcode, const uint32_t *args, uint32_t length) { switch (opcode) { @@ -16748,12 +18989,12 @@ bool CompilerMSL::SampledImageScanner::handle(spv::Op opcode, const uint32_t *ar return false; uint32_t result_type = args[0]; - auto &type = compiler.get(result_type); + auto &type = get(result_type); if ((type.basetype != SPIRType::Image && type.basetype != SPIRType::SampledImage) || type.image.sampled != 1) return true; uint32_t id = args[1]; - compiler.set(id, "", result_type, true); + set(id, "", result_type, true); break; } case OpImageSampleExplicitLod: @@ -16767,9 +19008,9 @@ bool CompilerMSL::SampledImageScanner::handle(spv::Op opcode, const uint32_t *ar case OpImageFetch: case OpImageGather: case OpImageDrefGather: - compiler.has_sampled_images = - compiler.has_sampled_images || compiler.is_sampled_image_type(compiler.expression_type(args[2])); - compiler.needs_swizzle_buffer_def = compiler.needs_swizzle_buffer_def || compiler.has_sampled_images; + self.has_sampled_images = + self.has_sampled_images || self.is_sampled_image_type(self.expression_type(args[2])); + self.needs_swizzle_buffer_def = self.needs_swizzle_buffer_def || self.has_sampled_images; break; default: break; @@ -16796,10 +19037,10 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui // suppress_missing_prototypes to suppress compiler warnings of missing function prototypes. // Mark if the input requires the implementation of an SPIR-V function that does not exist in Metal. - SPVFuncImpl spv_func = get_spv_func_impl(opcode, args); + SPVFuncImpl spv_func = get_spv_func_impl(opcode, args, length); if (spv_func != SPVFuncImplNone) { - compiler.spv_function_implementations.insert(spv_func); + self.spv_function_implementations.insert(spv_func); suppress_missing_prototypes = true; } @@ -16817,9 +19058,9 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui // Emulate texture2D atomic operations case OpImageTexelPointer: { - if (!compiler.msl_options.supports_msl_version(3, 1)) + if (!self.msl_options.supports_msl_version(3, 1)) { - auto *var = compiler.maybe_get_backing_variable(args[2]); + auto *var = self.maybe_get_backing_variable(args[2]); image_pointers_emulated[args[1]] = var ? var->self : ID(0); } break; @@ -16855,7 +19096,7 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui if (it != image_pointers_emulated.end()) { uses_image_write = true; - compiler.atomic_image_vars_emulated.insert(it->second); + self.atomic_image_vars_emulated.insert(it->second); } else check_resource_write(args[2]); @@ -16868,7 +19109,7 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui auto it = image_pointers_emulated.find(args[0]); if (it != image_pointers_emulated.end()) { - compiler.atomic_image_vars_emulated.insert(it->second); + self.atomic_image_vars_emulated.insert(it->second); uses_image_write = true; } else @@ -16882,7 +19123,7 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui auto it = image_pointers_emulated.find(args[2]); if (it != image_pointers_emulated.end()) { - compiler.atomic_image_vars_emulated.insert(it->second); + self.atomic_image_vars_emulated.insert(it->second); } break; } @@ -16903,13 +19144,46 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui needs_subgroup_invocation_id = true; break; + case OpGroupNonUniformRotateKHR: + // Add the correct invocation ID for calculating clustered rotate case. + if (length > 5) + { + if (static_cast(self.evaluate_constant_u32(args[2])) == ScopeSubgroup) + needs_subgroup_invocation_id = true; + else + needs_local_invocation_index = true; + } + break; + + case OpGroupNonUniformFAdd: + case OpGroupNonUniformFMul: + case OpGroupNonUniformFMin: + case OpGroupNonUniformFMax: + case OpGroupNonUniformIAdd: + case OpGroupNonUniformIMul: + case OpGroupNonUniformSMin: + case OpGroupNonUniformSMax: + case OpGroupNonUniformUMin: + case OpGroupNonUniformUMax: + case OpGroupNonUniformBitwiseAnd: + case OpGroupNonUniformBitwiseOr: + case OpGroupNonUniformBitwiseXor: + case OpGroupNonUniformLogicalAnd: + case OpGroupNonUniformLogicalOr: + case OpGroupNonUniformLogicalXor: + if ((compiler.get_execution_model() != ExecutionModelFragment || + self.msl_options.supports_msl_version(2, 2)) && + args[3] == GroupOperationClusteredReduce) + needs_subgroup_invocation_id = true; + break; + case OpArrayLength: { - auto *var = compiler.maybe_get_backing_variable(args[2]); + auto *var = self.maybe_get_backing_variable(args[2]); if (var != nullptr) { - if (!compiler.is_var_runtime_size_array(*var)) - compiler.buffers_requiring_array_length.insert(var->self); + if (!self.is_var_runtime_size_array(*var)) + self.buffers_requiring_array_length.insert(var->self); } break; } @@ -16923,16 +19197,39 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui uint32_t id = args[1]; uint32_t ptr = args[2]; - compiler.set(id, "", result_type, true); - compiler.register_read(id, ptr, true); - compiler.ir.ids[id].set_allow_type_rewrite(); + set(id, "", result_type, true); + self.register_read(id, ptr, true); + self.ir.ids[id].set_allow_type_rewrite(); + break; + } + + case OpBitcast: + case OpConvertPtrToU: + case OpConvertUToPtr: + { + if (length < 3) + break; + + auto &result_type = self.get(args[0]); + auto *arg_type = get_expression_result_type(args[2]); + if (!arg_type) + arg_type = &self.expression_type(args[2]); + + if (opcode != OpBitcast || self.is_pointer(result_type) || (arg_type && self.is_pointer(*arg_type))) + { + uint32_t id = args[1]; + set(id, "", args[0], true); + self.register_read(id, args[2], true); + self.ir.ids[id].set_allow_type_rewrite(); + } break; } case OpExtInst: { uint32_t extension_set = args[2]; - if (compiler.get(extension_set).ext == SPIRExtension::GLSL) + SPIRExtension::Extension ext = get(extension_set).ext; + if (ext == SPIRExtension::GLSL) { auto op_450 = static_cast(args[3]); switch (op_450) @@ -16941,15 +19238,15 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui case GLSLstd450InterpolateAtSample: case GLSLstd450InterpolateAtOffset: { - if (!compiler.msl_options.supports_msl_version(2, 3)) + if (!self.msl_options.supports_msl_version(2, 3)) SPIRV_CROSS_THROW("Pull-model interpolation requires MSL 2.3."); // Fragment varyings used with pull-model interpolation need special handling, // due to the way pull-model interpolation works in Metal. - auto *var = compiler.maybe_get_backing_variable(args[4]); + auto *var = self.maybe_get_backing_variable(args[4]); if (var) { - compiler.pull_model_inputs.insert(var->self); - auto &var_type = compiler.get_variable_element_type(*var); + self.pull_model_inputs.insert(var->self); + auto &var_type = self.get_variable_element_type(*var); // In addition, if this variable has a 'Sample' decoration, we need the sample ID // in order to do default interpolation. if (compiler.has_decoration(var->self, DecorationSample)) @@ -16975,50 +19272,74 @@ bool CompilerMSL::OpCodePreprocessor::handle(Op opcode, const uint32_t *args, ui break; } } + else if (ext == SPIRExtension::NonSemanticDebugPrintf) + { + // Operation 1 is printf. + if (args[3] == 1 && !self.msl_options.supports_msl_version(3, 2)) + SPIRV_CROSS_THROW("Debug printf requires MSL 3.2."); + } break; } case OpIsHelperInvocationEXT: - if (compiler.needs_manual_helper_invocation_updates()) + if (self.needs_manual_helper_invocation_updates()) needs_helper_invocation = true; break; + case OpCooperativeMatrixLoadKHR: + case OpCooperativeMatrixMulAddKHR: + case OpCooperativeMatrixLengthKHR: + uses_cooperative_matrix = true; + break; + + case OpCooperativeMatrixStoreKHR: + uses_cooperative_matrix = true; + check_resource_write(args[0]); + break; + default: break; } - // If it has one, keep track of the instruction's result type, mapped by ID - uint32_t result_type, result_id; - if (compiler.instruction_to_result_type(result_type, result_id, opcode, args, length)) - result_types[result_id] = result_type; - return true; } // If the variable is a Uniform or StorageBuffer, mark that a resource has been written to. void CompilerMSL::OpCodePreprocessor::check_resource_write(uint32_t var_id) { - auto *p_var = compiler.maybe_get_backing_variable(var_id); - StorageClass sc = p_var ? p_var->storage : StorageClassMax; - if (sc == StorageClassUniform || sc == StorageClassStorageBuffer) + auto *type = get_expression_result_type(var_id); + auto sc = StorageClassMax; + + if (type) + { + sc = type->storage; + } + else + { + auto *var = self.maybe_get_backing_variable(var_id); + if (var) + sc = var->storage; + } + + if (sc == StorageClassUniform || sc == StorageClassStorageBuffer || sc == StorageClassPhysicalStorageBuffer) uses_buffer_write = true; } // Returns an enumeration of a SPIR-V function that needs to be output for certain Op codes. -CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op opcode, const uint32_t *args) +CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op opcode, const uint32_t *args, uint32_t length) { switch (opcode) { + case OpSMod: + return SPVFuncImplSMod; + case OpFMod: return SPVFuncImplMod; case OpFAdd: case OpFSub: - if (compiler.msl_options.invariant_float_math || - compiler.has_decoration(args[1], DecorationNoContraction)) - { + if (self.msl_options.invariant_float_math || self.has_legacy_nocontract(args[0], args[1])) return opcode == OpFAdd ? SPVFuncImplFAdd : SPVFuncImplFSub; - } break; case OpFMul: @@ -17026,11 +19347,8 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o case OpMatrixTimesVector: case OpVectorTimesMatrix: case OpMatrixTimesMatrix: - if (compiler.msl_options.invariant_float_math || - compiler.has_decoration(args[1], DecorationNoContraction)) - { + if (self.msl_options.invariant_float_math || self.has_legacy_nocontract(args[0], args[1])) return SPVFuncImplFMul; - } break; case OpQuantizeToF16: @@ -17064,8 +19382,8 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o auto it = image_pointers_emulated.find(args[opcode == OpAtomicStore ? 0 : 2]); if (it != image_pointers_emulated.end()) { - uint32_t tid = compiler.get(it->second).basetype; - if (tid && compiler.get(tid).image.dim == Dim2D) + uint32_t tid = get(it->second).basetype; + if (tid && get(tid).image.dim == Dim2D) return SPVFuncImplImage2DAtomicCoords; } break; @@ -17077,7 +19395,7 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o { // Retrieve the image type, and if it's a Buffer, emit a texel coordinate function uint32_t tid = result_types[args[opcode == OpImageWrite ? 0 : 2]]; - if (tid && compiler.get(tid).image.dim == DimBuffer && !compiler.msl_options.texture_buffer_native) + if (tid && get(tid).image.dim == DimBuffer && !self.msl_options.texture_buffer_native) return SPVFuncImplTexelBufferCoords; break; } @@ -17085,7 +19403,7 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o case OpExtInst: { uint32_t extension_set = args[2]; - if (compiler.get(extension_set).ext == SPIRExtension::GLSL) + if (get(extension_set).ext == SPIRExtension::GLSL) { auto op_450 = static_cast(args[3]); switch (op_450) @@ -17104,28 +19422,28 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o return SPVFuncImplSSign; case GLSLstd450Reflect: { - auto &type = compiler.get(args[0]); + auto &type = get(args[0]); if (type.vecsize == 1) return SPVFuncImplReflectScalar; break; } case GLSLstd450Refract: { - auto &type = compiler.get(args[0]); + auto &type = get(args[0]); if (type.vecsize == 1) return SPVFuncImplRefractScalar; break; } case GLSLstd450FaceForward: { - auto &type = compiler.get(args[0]); + auto &type = get(args[0]); if (type.vecsize == 1) return SPVFuncImplFaceForwardScalar; break; } case GLSLstd450MatrixInverse: { - auto &mat_type = compiler.get(args[0]); + auto &mat_type = get(args[0]); switch (mat_type.columns) { case 2: @@ -17187,12 +19505,34 @@ CompilerMSL::SPVFuncImpl CompilerMSL::OpCodePreprocessor::get_spv_func_impl(Op o case OpGroupNonUniformShuffleDown: return SPVFuncImplSubgroupShuffleDown; + case OpGroupNonUniformRotateKHR: + // Clustered rotate is performed using shuffle. + if (length > 5) + return SPVFuncImplSubgroupShuffle; + return SPVFuncImplSubgroupRotate; + case OpGroupNonUniformQuadBroadcast: return SPVFuncImplQuadBroadcast; case OpGroupNonUniformQuadSwap: return SPVFuncImplQuadSwap; + case OpSDot: + case OpUDot: + case OpSUDot: + case OpSDotAccSat: + case OpUDotAccSat: + case OpSUDotAccSat: + return SPVFuncImplReduceAdd; + + case OpSMulExtended: + case OpUMulExtended: + return SPVFuncImplMulExtended; + + case OpAssumeTrueKHR: + case OpExpectKHR: + return SPVFuncImplAssume; + default: break; } @@ -17376,6 +19716,7 @@ void CompilerMSL::cast_from_variable_load(uint32_t source_id, std::string &expr, case BuiltInSubgroupSize: case BuiltInSubgroupLocalInvocationId: case BuiltInViewIndex: + case BuiltInDrawIndex: case BuiltInVertexIndex: case BuiltInInstanceIndex: case BuiltInBaseInstance: @@ -17588,6 +19929,108 @@ bool CompilerMSL::is_supported_argument_buffer_type(const SPIRType &type) const return is_supported_type && !type_is_msl_framebuffer_fetch(type); } +void CompilerMSL::emit_argument_buffer_aliased_descriptor(const SPIRVariable &aliased_var, + const SPIRVariable &base_var) +{ + // To deal with buffer <-> image aliasing, we need to perform an unholy UB ritual. + // A texture type in Metal 3.0 is a pointer. However, we cannot simply cast a pointer to texture. + // What we *can* do is to cast pointer-to-pointer to pointer-to-texture. + + // We need to explicitly reach into the descriptor buffer lvalue, not any spvDescriptorArray wrapper. + auto *var_meta = ir.find_meta(base_var.self); + bool old_explicit_qualifier = var_meta && var_meta->decoration.qualified_alias_explicit_override; + if (var_meta) + var_meta->decoration.qualified_alias_explicit_override = false; + auto unqualified_name = to_name(base_var.self, false); + if (var_meta) + var_meta->decoration.qualified_alias_explicit_override = old_explicit_qualifier; + + // For non-arrayed buffers, we have already performed a de-reference. + // We need a proper lvalue to cast, so strip away the de-reference. + if (unqualified_name.size() > 2 && unqualified_name[0] == '(' && unqualified_name[1] == '*') + { + unqualified_name.erase(unqualified_name.begin(), unqualified_name.begin() + 2); + unqualified_name.pop_back(); + } + + string name; + + auto &var_type = get(aliased_var.basetype); + auto &data_type = get_variable_data_type(aliased_var); + string descriptor_storage = descriptor_address_space(aliased_var.self, aliased_var.storage, ""); + + if (aliased_var.storage == StorageClassUniformConstant) + { + if (is_var_runtime_size_array(aliased_var)) + { + // This becomes a plain pointer to spvDescriptor. + name = join("reinterpret_cast<", descriptor_storage, " ", + type_to_glsl(get_variable_data_type(aliased_var), aliased_var.self, true), ">(&", + unqualified_name, ")"); + } + else + { + name = join("reinterpret_cast<", descriptor_storage, " ", + type_to_glsl(get_variable_data_type(aliased_var), aliased_var.self, true), " &>(", + unqualified_name, ");"); + } + } + else + { + // Buffer types. + bool old_is_using_builtin_array = is_using_builtin_array; + is_using_builtin_array = true; + + bool needs_post_cast_deref = !is_array(data_type); + string ref_type = needs_post_cast_deref ? "&" : join("(&)", type_to_array_glsl(var_type, aliased_var.self)); + + if (is_var_runtime_size_array(aliased_var)) + { + name = join("reinterpret_cast<", + type_to_glsl(var_type, aliased_var.self, true), " ", descriptor_storage, " *>(&", + unqualified_name, ")"); + } + else + { + name = join(needs_post_cast_deref ? "*" : "", "reinterpret_cast<", + type_to_glsl(var_type, aliased_var.self, true), " ", descriptor_storage, " ", + ref_type, + ">(", unqualified_name, ");"); + } + + if (needs_post_cast_deref) + descriptor_storage = get_type_address_space(var_type, aliased_var.self, false); + + // These kinds of ridiculous casts trigger warnings in compiler. Just ignore them. + if (!suppress_incompatible_pointer_types_discard_qualifiers) + { + suppress_incompatible_pointer_types_discard_qualifiers = true; + force_recompile_guarantee_forward_progress(); + } + + is_using_builtin_array = old_is_using_builtin_array; + } + + if (!is_var_runtime_size_array(aliased_var)) + { + // Lower to temporary, so drop the qualification. + set_qualified_name(aliased_var.self, ""); + statement(descriptor_storage, " auto &", to_name(aliased_var.self), " = ", name); + } + else + { + // This alias may have already been used to emit an entry point declaration. If there is a mismatch, we need a recompile. + // Moving this code to be run earlier will also conflict, + // because we need the qualified alias for the base resource, + // so forcing recompile until things sync up is the least invasive method for now. + if (ir.meta[aliased_var.self].decoration.qualified_alias != name) + force_recompile(); + + // This will get wrapped in a separate temporary when a spvDescriptorArray wrapper is emitted. + set_qualified_name(aliased_var.self, name); + } +} + void CompilerMSL::analyze_argument_buffers() { // Gather all used resources and sort them out into argument buffers. @@ -17604,11 +20047,12 @@ void CompilerMSL::analyze_argument_buffers() struct Resource { SPIRVariable *var; - SPIRVariable *descriptor_alias; string name; SPIRType::BaseType basetype; uint32_t index; + uint32_t plane_count; uint32_t plane; + uint32_t overlapping_var_id; }; SmallVector resources_in_set[kMaxArgumentBuffers]; SmallVector inline_block_vars; @@ -17644,32 +20088,6 @@ void CompilerMSL::analyze_argument_buffers() } } - // Handle descriptor aliasing as well as we can. - // We can handle aliasing of buffers by casting pointers, but not for typed resources. - // Inline UBOs cannot be handled since it's not a pointer, but inline data. - SPIRVariable *descriptor_alias = nullptr; - if (var.storage == StorageClassUniform || var.storage == StorageClassStorageBuffer) - { - for (auto &resource : resources_in_set[desc_set]) - { - if (get_decoration(resource.var->self, DecorationBinding) == - get_decoration(var_id, DecorationBinding) && - resource.basetype == SPIRType::Struct && type.basetype == SPIRType::Struct && - (resource.var->storage == StorageClassUniform || - resource.var->storage == StorageClassStorageBuffer)) - { - descriptor_alias = resource.var; - // Self-reference marks that we should declare the resource, - // and it's being used as an alias (so we can emit void* instead). - resource.descriptor_alias = resource.var; - // Need to promote interlocked usage so that the primary declaration is correct. - if (interlocked_resources.count(var_id)) - interlocked_resources.insert(resource.var->self); - break; - } - } - } - uint32_t binding = get_decoration(var_id, DecorationBinding); if (type.basetype == SPIRType::SampledImage) { @@ -17683,14 +20101,14 @@ void CompilerMSL::analyze_argument_buffers() { uint32_t image_resource_index = get_metal_resource_index(var, SPIRType::Image, i); resources_in_set[desc_set].push_back( - { &var, descriptor_alias, to_name(var_id), SPIRType::Image, image_resource_index, i }); + { &var, to_name(var_id), SPIRType::Image, image_resource_index, plane_count, i, 0 }); } if (type.image.dim != DimBuffer && !constexpr_sampler) { uint32_t sampler_resource_index = get_metal_resource_index(var, SPIRType::Sampler); resources_in_set[desc_set].push_back( - { &var, descriptor_alias, to_sampler_expression(var_id), SPIRType::Sampler, sampler_resource_index, 0 }); + { &var, to_sampler_expression(var_id), SPIRType::Sampler, sampler_resource_index, 1, 0, 0 }); } } else if (inline_uniform_blocks.count(SetBindingPair{ desc_set, binding })) @@ -17703,19 +20121,17 @@ void CompilerMSL::analyze_argument_buffers() // Inline uniform blocks are always emitted at the end. add_resource_name(var_id); - uint32_t resource_index = ~0u; - if (!descriptor_alias) - resource_index = get_metal_resource_index(var, type.basetype); + uint32_t resource_index = get_metal_resource_index(var, type.basetype); resources_in_set[desc_set].push_back( - { &var, descriptor_alias, to_name(var_id), type.basetype, resource_index, 0 }); + { &var, to_name(var_id), type.basetype, resource_index, 1, 0, 0 }); // Emulate texture2D atomic operations if (atomic_image_vars_emulated.count(var.self)) { uint32_t buffer_resource_index = get_metal_resource_index(var, SPIRType::AtomicCounter, 0); resources_in_set[desc_set].push_back( - { &var, descriptor_alias, to_name(var_id) + "_atomic", SPIRType::Struct, buffer_resource_index, 0 }); + { &var, to_name(var_id) + "_atomic", SPIRType::Struct, buffer_resource_index, 1, 0, 0 }); } } @@ -17763,7 +20179,7 @@ void CompilerMSL::analyze_argument_buffers() set_decoration(var_id, DecorationDescriptorSet, desc_set); set_decoration(var_id, DecorationBinding, kSwizzleBufferBinding); resources_in_set[desc_set].push_back( - { &var, nullptr, to_name(var_id), SPIRType::UInt, get_metal_resource_index(var, SPIRType::UInt), 0 }); + { &var, to_name(var_id), SPIRType::UInt, get_metal_resource_index(var, SPIRType::UInt), 1, 0, 0 }); } if (set_needs_buffer_sizes[desc_set]) @@ -17774,7 +20190,7 @@ void CompilerMSL::analyze_argument_buffers() set_decoration(var_id, DecorationDescriptorSet, desc_set); set_decoration(var_id, DecorationBinding, kBufferSizeBufferBinding); resources_in_set[desc_set].push_back( - { &var, nullptr, to_name(var_id), SPIRType::UInt, get_metal_resource_index(var, SPIRType::UInt), 0 }); + { &var, to_name(var_id), SPIRType::UInt, get_metal_resource_index(var, SPIRType::UInt), 1, 0, 0 }); } } } @@ -17786,7 +20202,7 @@ void CompilerMSL::analyze_argument_buffers() uint32_t desc_set = get_decoration(var_id, DecorationDescriptorSet); add_resource_name(var_id); resources_in_set[desc_set].push_back( - { &var, nullptr, to_name(var_id), SPIRType::Struct, get_metal_resource_index(var, SPIRType::Struct), 0 }); + { &var, to_name(var_id), SPIRType::Struct, get_metal_resource_index(var, SPIRType::Struct), 1, 0, 0 }); } for (uint32_t desc_set = 0; desc_set < kMaxArgumentBuffers; desc_set++) @@ -17817,82 +20233,115 @@ void CompilerMSL::analyze_argument_buffers() else buffer_type.storage = StorageClassUniform; - set_name(type_id, join("spvDescriptorSetBuffer", desc_set)); + auto buffer_type_name = join("spvDescriptorSetBuffer", desc_set); + set_name(type_id, buffer_type_name); auto &ptr_type = set(ptr_type_id, OpTypePointer); ptr_type = buffer_type; - ptr_type.op = spv::OpTypePointer; + ptr_type.op = OpTypePointer; ptr_type.pointer = true; ptr_type.pointer_depth++; ptr_type.parent_type = type_id; uint32_t buffer_variable_id = next_id; - set(buffer_variable_id, ptr_type_id, StorageClassUniform); - set_name(buffer_variable_id, join("spvDescriptorSet", desc_set)); + auto &buffer_var = set(buffer_variable_id, ptr_type_id, StorageClassUniform); + auto buffer_name = join("spvDescriptorSet", desc_set); + set_name(buffer_variable_id, buffer_name); // Ids must be emitted in ID order. stable_sort(begin(resources), end(resources), [&](const Resource &lhs, const Resource &rhs) -> bool { return tie(lhs.index, lhs.basetype) < tie(rhs.index, rhs.basetype); }); + for (size_t i = 0; i < resources.size() - 1; i++) + { + auto &r1 = resources[i]; + auto &r2 = resources[i + 1]; + + if (r1.index == r2.index) + { + if (r1.overlapping_var_id) + r2.overlapping_var_id = r1.overlapping_var_id; + else + r2.overlapping_var_id = r1.var->self; + + set_extended_decoration(r2.var->self, SPIRVCrossDecorationOverlappingBinding, r2.overlapping_var_id); + } + } + uint32_t member_index = 0; uint32_t next_arg_buff_index = 0; + uint32_t prev_was_scalar_on_array_offset = 0; for (auto &resource : resources) { auto &var = *resource.var; auto &type = get_variable_data_type(var); + if (is_var_runtime_size_array(var) && (argument_buffer_device_storage_mask & (1u << desc_set)) == 0) + SPIRV_CROSS_THROW("Runtime sized variables must be in device storage argument buffers."); + // If needed, synthesize and add padding members. // member_index and next_arg_buff_index are incremented when padding members are added. - if (msl_options.pad_argument_buffer_resources) + if (msl_options.pad_argument_buffer_resources && resource.plane == 0 && resource.overlapping_var_id == 0) { - auto &rez_bind = get_argument_buffer_resource(desc_set, next_arg_buff_index); - if (!resource.descriptor_alias) + auto rez_bind = get_argument_buffer_resource(desc_set, next_arg_buff_index - prev_was_scalar_on_array_offset); + rez_bind.count -= prev_was_scalar_on_array_offset; + + while (resource.index > next_arg_buff_index) { - while (resource.index > next_arg_buff_index) + switch (rez_bind.basetype) { - switch (rez_bind.basetype) - { - case SPIRType::Void: - case SPIRType::Boolean: - case SPIRType::SByte: - case SPIRType::UByte: - case SPIRType::Short: - case SPIRType::UShort: - case SPIRType::Int: - case SPIRType::UInt: - case SPIRType::Int64: - case SPIRType::UInt64: - case SPIRType::AtomicCounter: - case SPIRType::Half: - case SPIRType::Float: - case SPIRType::Double: - add_argument_buffer_padding_buffer_type(buffer_type, member_index, next_arg_buff_index, rez_bind); - break; - case SPIRType::Image: - add_argument_buffer_padding_image_type(buffer_type, member_index, next_arg_buff_index, rez_bind); - break; - case SPIRType::Sampler: + case SPIRType::Void: + case SPIRType::Boolean: + case SPIRType::SByte: + case SPIRType::UByte: + case SPIRType::Short: + case SPIRType::UShort: + case SPIRType::Int: + case SPIRType::UInt: + case SPIRType::Int64: + case SPIRType::UInt64: + case SPIRType::AtomicCounter: + case SPIRType::Half: + case SPIRType::Float: + case SPIRType::Double: + add_argument_buffer_padding_buffer_type(buffer_type, member_index, next_arg_buff_index, rez_bind); + break; + case SPIRType::Image: + add_argument_buffer_padding_image_type(buffer_type, member_index, next_arg_buff_index, rez_bind); + break; + case SPIRType::Sampler: + add_argument_buffer_padding_sampler_type(buffer_type, member_index, next_arg_buff_index, rez_bind); + break; + case SPIRType::SampledImage: + if (next_arg_buff_index == rez_bind.msl_sampler) add_argument_buffer_padding_sampler_type(buffer_type, member_index, next_arg_buff_index, rez_bind); - break; - case SPIRType::SampledImage: - if (next_arg_buff_index == rez_bind.msl_sampler) - add_argument_buffer_padding_sampler_type(buffer_type, member_index, next_arg_buff_index, rez_bind); - else - add_argument_buffer_padding_image_type(buffer_type, member_index, next_arg_buff_index, rez_bind); - break; - default: - break; - } + else + add_argument_buffer_padding_image_type(buffer_type, member_index, next_arg_buff_index, rez_bind); + break; + default: + break; } + + // After padding, retrieve the resource again. It will either be more padding, or the actual resource. + rez_bind = get_argument_buffer_resource(desc_set, next_arg_buff_index); + prev_was_scalar_on_array_offset = 0; } + uint32_t count = rez_bind.count; + + // If the current resource is an array in the descriptor, but is a scalar + // in the shader, only the first element will be consumed. The next pass + // will add a padding member to consume the remaining array elements. + if (count > 1 && type.array.empty()) + count = prev_was_scalar_on_array_offset = 1; + // Adjust the number of slots consumed by current member itself. - // Use the count value from the app, instead of the shader, in case the - // shader is only accesing part, or even one element, of the array. - next_arg_buff_index += rez_bind.count; + next_arg_buff_index += resource.plane_count * count; } + // Here we're locking down the member name early before compilation loops, so ensure that + // the resource name is not reused, even through a reset(). string mbr_name = ensure_valid_name(resource.name, "m"); if (resource.plane > 0) mbr_name += join(plane_name_suffix, resource.plane); @@ -17927,27 +20376,35 @@ void CompilerMSL::analyze_argument_buffers() SetBindingPair pair = { desc_set, binding }; if (resource.basetype == SPIRType::Image || resource.basetype == SPIRType::Sampler || - resource.basetype == SPIRType::SampledImage) + resource.basetype == SPIRType::SampledImage || resource.basetype == SPIRType::AccelerationStructure) { // Drop pointer information when we emit the resources into a struct. buffer_type.member_types.push_back(get_variable_data_type_id(var)); - if (resource.plane == 0) + if (has_extended_decoration(var.self, SPIRVCrossDecorationOverlappingBinding)) + { + if (!msl_options.supports_msl_version(3, 0)) + SPIRV_CROSS_THROW("Full mutable aliasing of argument buffer descriptors only works on Metal 3+."); + + auto &entry_func = get(ir.default_entry_point); + entry_func.fixup_hooks_in.push_back([this, resource]() { + emit_argument_buffer_aliased_descriptor(*resource.var, this->get(resource.overlapping_var_id)); + }); + } + else if (resource.plane == 0) + { set_qualified_name(var.self, join(to_name(buffer_variable_id), ".", mbr_name)); + } } else if (buffers_requiring_dynamic_offset.count(pair)) { - if (resource.descriptor_alias) - SPIRV_CROSS_THROW("Descriptor aliasing is currently not supported with dynamic offsets."); - // Don't set the qualified name here; we'll define a variable holding the corrected buffer address later. buffer_type.member_types.push_back(var.basetype); - buffers_requiring_dynamic_offset[pair].second = var.self; + auto &dynamic_buffer = buffers_requiring_dynamic_offset[pair]; + dynamic_buffer.var_id = var.self; + dynamic_buffer.mbr_name = mbr_name; } else if (inline_uniform_blocks.count(pair)) { - if (resource.descriptor_alias) - SPIRV_CROSS_THROW("Descriptor aliasing is currently not supported with inline UBOs."); - // Put the buffer block itself into the argument buffer. buffer_type.member_types.push_back(get_variable_data_type_id(var)); set_qualified_name(var.self, join(to_name(buffer_variable_id), ".", mbr_name)); @@ -17980,11 +20437,22 @@ void CompilerMSL::analyze_argument_buffers() } else { - if (!resource.descriptor_alias || resource.descriptor_alias == resource.var) - buffer_type.member_types.push_back(var.basetype); + buffer_type.member_types.push_back(var.basetype); + if (has_extended_decoration(var.self, SPIRVCrossDecorationOverlappingBinding)) + { + // Casting raw pointers is fine since their ABI is fixed, but anything opaque is deeply questionable on Metal 2. + if (get(resource.overlapping_var_id).storage == StorageClassUniformConstant && + !msl_options.supports_msl_version(3, 0)) + { + SPIRV_CROSS_THROW("Full mutable aliasing of argument buffer descriptors only works on Metal 3+."); + } - if (resource.descriptor_alias && resource.descriptor_alias != resource.var) - buffer_aliases_argument.push_back({ var.self, resource.descriptor_alias->self }); + auto &entry_func = get(ir.default_entry_point); + + entry_func.fixup_hooks_in.push_back([this, resource]() { + emit_argument_buffer_aliased_descriptor(*resource.var, this->get(resource.overlapping_var_id)); + }); + } else if (type.array.empty()) set_qualified_name(var.self, join("(*", to_name(buffer_variable_id), ".", mbr_name, ")")); else @@ -17996,8 +20464,20 @@ void CompilerMSL::analyze_argument_buffers() resource.index); set_extended_member_decoration(buffer_type.self, member_index, SPIRVCrossDecorationInterfaceOrigID, var.self); + if (has_extended_decoration(var.self, SPIRVCrossDecorationOverlappingBinding)) + set_extended_member_decoration(buffer_type.self, member_index, SPIRVCrossDecorationOverlappingBinding); member_index++; } + + if (msl_options.replace_recursive_inputs && type_contains_recursion(buffer_type)) + { + recursive_inputs.insert(type_id); + auto &entry_func = this->get(ir.default_entry_point); + auto addr_space = get_variable_address_space(buffer_var); + entry_func.fixup_hooks_in.push_back([this, addr_space, buffer_name, buffer_type_name]() { + statement(addr_space, " auto& ", buffer_name, " = *(", addr_space, " ", buffer_type_name, "*)", buffer_name, "_vp;"); + }); + } } } @@ -18005,7 +20485,7 @@ void CompilerMSL::analyze_argument_buffers() // that matches the resource index of the argument buffer index. // This is a two-step lookup, first lookup the resource binding number from the argument buffer index, // then lookup the resource binding using the binding number. -MSLResourceBinding &CompilerMSL::get_argument_buffer_resource(uint32_t desc_set, uint32_t arg_idx) +const MSLResourceBinding &CompilerMSL::get_argument_buffer_resource(uint32_t desc_set, uint32_t arg_idx) const { auto stage = get_entry_point().model; StageSetBinding arg_idx_tuple = { stage, desc_set, arg_idx }; @@ -18036,7 +20516,7 @@ void CompilerMSL::add_argument_buffer_padding_buffer_type(SPIRType &struct_type, uint32_t ptr_type_id = buff_type_id + 1; auto &ptr_type = set(ptr_type_id, OpTypePointer); ptr_type = buff_type; - ptr_type.op = spv::OpTypePointer; + ptr_type.op = OpTypePointer; ptr_type.pointer = true; ptr_type.pointer_depth++; ptr_type.parent_type = buff_type_id; @@ -18148,10 +20628,321 @@ const char *CompilerMSL::get_combined_sampler_suffix() const return sampler_name_suffix.c_str(); } +bool CompilerMSL::specialization_constant_is_macro(uint32_t const_id) const +{ + return constant_macro_ids.find(const_id) != constant_macro_ids.end(); +} + +// Start with all fast math flags enabled, and selectively disable based execution modes and float controls +uint32_t CompilerMSL::get_fp_fast_math_flags(bool incl_ops) const +{ + uint32_t fp_flags = ~0; + auto &ep = get_entry_point(); + + if (ep.flags.get(ExecutionModeSignedZeroInfNanPreserve)) + fp_flags &= ~(FPFastMathModeNSZMask | FPFastMathModeNotInfMask | FPFastMathModeNotNaNMask); + + if (ep.flags.get(ExecutionModeContractionOff)) + fp_flags &= ~(FPFastMathModeAllowContractMask); + + for (auto &fp_pair : ep.fp_fast_math_defaults) + if (fp_pair.second) + fp_flags &= get(fp_pair.second).scalar(); + + if (incl_ops) + for (auto &p_m : ir.meta) + if (p_m.second.decoration.decoration_flags.get(DecorationFPFastMathMode)) + fp_flags &= p_m.second.decoration.fp_fast_math_mode; + + return fp_flags; +} + void CompilerMSL::emit_block_hints(const SPIRBlock &) { } +void CompilerMSL::emit_mesh_entry_point() +{ + auto &ep = get_entry_point(); + auto &f = get(ir.default_entry_point); + + const uint32_t func_id = ir.increase_bound_by(3); + const uint32_t block_id = func_id + 1; + const uint32_t ret_id = func_id + 2; + auto &wrapped_main = set(func_id, f.return_type, f.function_type); + + wrapped_main.blocks.push_back(block_id); + wrapped_main.entry_block = block_id; + + auto &wrapped_entry = set(block_id); + wrapped_entry.terminator = SPIRBlock::Return; + + // Push call to original 'main' + Instruction ix = {}; + ix.op = OpFunctionCall; + ix.offset = uint32_t(ir.spirv.size()); + ix.length = 3; + + ir.spirv.push_back(f.return_type); + ir.spirv.push_back(ret_id); + ir.spirv.push_back(ep.self); + + wrapped_entry.ops.push_back(ix); + + // relace entry-point for new one + SPIREntryPoint proxy_ep = ep; + proxy_ep.self = func_id; + ir.entry_points.insert(std::make_pair(func_id, proxy_ep)); + ir.meta[func_id] = ir.meta[ir.default_entry_point]; + ir.meta[ir.default_entry_point].decoration.alias.clear(); + + ir.default_entry_point = func_id; +} + +void CompilerMSL::emit_mesh_outputs() +{ + auto &mode = get_entry_point(); + + // predefined thread count or zero, if specialization constant is in use + uint32_t num_invocations = 0; + if (mode.workgroup_size.id_x == 0 && mode.workgroup_size.id_y == 0 && mode.workgroup_size.id_z == 0) + num_invocations = mode.workgroup_size.x * mode.workgroup_size.y * mode.workgroup_size.z; + + statement("threadgroup_barrier(mem_flags::mem_threadgroup);"); + statement("if (spvMeshSizes.y == 0)"); + begin_scope(); + statement("return;"); + end_scope(); + statement("spvMesh.set_primitive_count(spvMeshSizes.y);"); + + statement("const uint spvThreadCount [[maybe_unused]] = (gl_WorkGroupSize.x * gl_WorkGroupSize.y * gl_WorkGroupSize.z);"); + + if (mesh_out_per_vertex != 0) + { + auto &type_vert = get(mesh_out_per_vertex); + + if (num_invocations < mode.output_vertices) + { + statement("for (uint spvVI = gl_LocalInvocationIndex; spvVI < spvMeshSizes.x; spvVI += spvThreadCount)"); + } + else + { + statement("const uint spvVI = gl_LocalInvocationIndex;"); + statement("if (gl_LocalInvocationIndex < spvMeshSizes.x)"); + } + + begin_scope(); + + statement("spvPerVertex spvV = {};"); + for (uint32_t index = 0; index < uint32_t(type_vert.member_types.size()); ++index) + { + uint32_t orig_var = get_extended_member_decoration(type_vert.self, index, SPIRVCrossDecorationInterfaceOrigID); + uint32_t orig_id = get_extended_member_decoration(type_vert.self, index, SPIRVCrossDecorationInterfaceMemberIndex); + + // Clip/cull distances are special-case + if (orig_var == 0 && orig_id == (~0u)) + continue; + + auto &orig = get(orig_var); + auto &orig_type = get(orig.basetype); + + // FIXME: Need to deal with complex composite IO types. These may need extra unroll, etc. + + BuiltIn builtin = BuiltInMax; + std::string access; + if (orig_type.basetype == SPIRType::Struct) + { + if (has_member_decoration(orig_type.self, orig_id, DecorationBuiltIn)) + builtin = BuiltIn(get_member_decoration(orig_type.self, orig_id, DecorationBuiltIn)); + + switch (builtin) + { + case BuiltInPosition: + case BuiltInPointSize: + case BuiltInClipDistance: + case BuiltInCullDistance: + access = "." + builtin_to_glsl(builtin, StorageClassOutput); + break; + default: + access = "." + to_member_name(orig_type, orig_id); + break; + } + + if (has_member_decoration(type_vert.self, index, DecorationIndex)) + { + // Declare the Clip/CullDistance as [[user(clip/cullN)]]. + const uint32_t orig_index = get_member_decoration(type_vert.self, index, DecorationIndex); + access += "[" + to_string(orig_index) + "]"; + statement("spvV.", builtin_to_glsl(builtin, StorageClassOutput), "[", orig_index, "] = ", to_name(orig_var), "[spvVI]", access, ";"); + } + } + + statement("spvV.", to_member_name(type_vert, index), " = ", to_name(orig_var), "[spvVI]", access, ";"); + if (options.vertex.flip_vert_y && builtin == BuiltInPosition) + { + statement("spvV.", to_member_name(type_vert, index), ".y = -(", "spvV.", + to_member_name(type_vert, index), ".y);", " // Invert Y-axis for Metal"); + } + } + statement("spvMesh.set_vertex(spvVI, spvV);"); + end_scope(); + } + + if (mesh_out_per_primitive != 0 || builtin_mesh_primitive_indices_id != 0) + { + if (num_invocations < mode.output_primitives) + { + statement("for (uint spvPI = gl_LocalInvocationIndex; spvPI < spvMeshSizes.y; spvPI += spvThreadCount)"); + } + else + { + statement("const uint spvPI = gl_LocalInvocationIndex;"); + statement("if (gl_LocalInvocationIndex < spvMeshSizes.y)"); + } + + // FIXME: Need to deal with complex composite IO types. These may need extra unroll, etc. + + begin_scope(); + + if (builtin_mesh_primitive_indices_id != 0) + { + if (mode.flags.get(ExecutionModeOutputTrianglesEXT)) + { + statement("spvMesh.set_index(spvPI * 3u + 0u, gl_PrimitiveTriangleIndicesEXT[spvPI].x);"); + statement("spvMesh.set_index(spvPI * 3u + 1u, gl_PrimitiveTriangleIndicesEXT[spvPI].y);"); + statement("spvMesh.set_index(spvPI * 3u + 2u, gl_PrimitiveTriangleIndicesEXT[spvPI].z);"); + } + else if (mode.flags.get(ExecutionModeOutputLinesEXT)) + { + statement("spvMesh.set_index(spvPI * 2u + 0u, gl_PrimitiveLineIndicesEXT[spvPI].x);"); + statement("spvMesh.set_index(spvPI * 2u + 1u, gl_PrimitiveLineIndicesEXT[spvPI].y);"); + } + else + { + statement("spvMesh.set_index(spvPI, gl_PrimitivePointIndicesEXT[spvPI]);"); + } + } + + if (mesh_out_per_primitive != 0) + { + auto &type_prim = get(mesh_out_per_primitive); + statement("spvPerPrimitive spvP = {};"); + for (uint32_t index = 0; index < uint32_t(type_prim.member_types.size()); ++index) + { + uint32_t orig_var = + get_extended_member_decoration(type_prim.self, index, SPIRVCrossDecorationInterfaceOrigID); + uint32_t orig_id = + get_extended_member_decoration(type_prim.self, index, SPIRVCrossDecorationInterfaceMemberIndex); + auto &orig = get(orig_var); + auto &orig_type = get(orig.basetype); + + BuiltIn builtin = BuiltInMax; + std::string access; + if (orig_type.basetype == SPIRType::Struct) + { + if (has_member_decoration(orig_type.self, orig_id, DecorationBuiltIn)) + builtin = BuiltIn(get_member_decoration(orig_type.self, orig_id, DecorationBuiltIn)); + + switch (builtin) + { + case BuiltInPrimitiveId: + case BuiltInLayer: + case BuiltInViewportIndex: + case BuiltInCullPrimitiveEXT: + case BuiltInPrimitiveShadingRateKHR: + access = "." + builtin_to_glsl(builtin, StorageClassOutput); + break; + default: + access = "." + to_member_name(orig_type, orig_id); + } + } + statement("spvP.", to_member_name(type_prim, index), " = ", to_name(orig_var), "[spvPI]", access, ";"); + } + statement("spvMesh.set_primitive(spvPI, spvP);"); + } + + end_scope(); + } +} + +void CompilerMSL::emit_mesh_tasks(SPIRBlock &block) +{ + // GLSL: Once this instruction is called, the workgroup must be terminated immediately, and the mesh shaders are launched. + // TODO: find relieble and clean of terminating shader. + flush_variable_declaration(builtin_task_grid_id); + statement("spvMgp.set_threadgroups_per_grid(uint3(", to_unpacked_expression(block.mesh.groups[0]), ", ", + to_unpacked_expression(block.mesh.groups[1]), ", ", to_unpacked_expression(block.mesh.groups[2]), "));"); + // This is correct if EmitMeshTasks is called in the entry function for shader. + // Only viable solutions would be: + // - Caller ensures the SPIR-V is inlined, then this always holds true. + // - Pass down a "should terminate" bool to leaf functions and chain return (horrible and disgusting, let's not). + statement("return;"); +} + +void CompilerMSL::emit_workgroup_initialization(const SPIRVariable &var) +{ + auto &type = get_variable_data_type(var); + + begin_scope(); + + if (type.array.empty() && type.member_types.empty()) + { + // For simple shared variables, we just initialize it in thread 0 of the block + // We use short to represent bool for threadgroup variable to workaround compiler bug, + // so we do a temporary fixup here. Alas. (see the type_to_glsl method) + bool is_boolean = type.basetype == SPIRType::Boolean; + if (is_boolean) + type.basetype = SPIRType::Short; + + statement("if (gl_LocalInvocationIndex == 0)"); + begin_scope(); + statement(to_name(var.self), " = ", to_initializer_expression(var), ";"); + end_scope(); + + if (is_boolean) + type.basetype = SPIRType::Boolean; + } + else + { + // Otherwise, we use a loop to cooperatively initialize the memory within the group + + // First, we define a few variable names; + string var_name = to_name(var.self); + string var_ptr_name = join(var_name, "_ptr"); + string var_size_name = join(var_name, "_sz"); + string var_pos_name = join(var_name, "_pos"); + string var_stride_name = join(var_name, "_stride"); + string var_ptr2_name = join(var_name, "_ptr2"); + + statement("threadgroup uint *", var_ptr_name, " = (threadgroup uint *)&", var_name, ";"); + statement("uint ", var_size_name, " = ", "sizeof(", var_name, ");"); + statement("uint ", var_pos_name, " = gl_LocalInvocationIndex;"); + statement("uint ", var_stride_name, " = gl_WorkGroupSize.x * gl_WorkGroupSize.y * gl_WorkGroupSize.z;"); + + statement("while (sizeof(uint) * ", var_pos_name, " < ", var_size_name, ")"); + begin_scope(); + statement(var_ptr_name, "[", var_pos_name, "] = 0u;"); + statement(var_pos_name, " += ", var_stride_name, ";"); + end_scope(); + + statement("if (gl_LocalInvocationIndex == 0)"); + begin_scope(); + statement(var_pos_name, " = (", var_size_name, " / sizeof(uint)) * sizeof(uint);"); + statement("threadgroup uchar *", var_ptr2_name, " = (threadgroup uchar *)&", var_name, ";"); + + statement("while (", var_pos_name, " < ", var_size_name, ")"); + begin_scope(); + statement(var_ptr2_name, "[", var_pos_name, "] = '\\0';"); + statement(var_pos_name, "++;"); + end_scope(); + end_scope(); + } + + statement("threadgroup_barrier(mem_flags::mem_threadgroup);"); + + end_scope(); +} + string CompilerMSL::additional_fixed_sample_mask_str() const { char print_buffer[32]; diff --git a/src/libraries/spirv_cross/spirv_msl.hpp b/src/libraries/spirv_cross/spirv_msl.hpp index 45109d88f..de8608f65 100644 --- a/src/libraries/spirv_cross/spirv_msl.hpp +++ b/src/libraries/spirv_cross/spirv_msl.hpp @@ -33,6 +33,7 @@ namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; // Indicates the format of a shader interface variable. Currently limited to specifying // if the input is an 8-bit unsigned integer, 16-bit unsigned integer, or @@ -78,7 +79,7 @@ struct MSLShaderInterfaceVariable uint32_t location = 0; uint32_t component = 0; MSLShaderVariableFormat format = MSL_SHADER_VARIABLE_FORMAT_OTHER; - spv::BuiltIn builtin = spv::BuiltInMax; + BuiltIn builtin = BuiltInMax; uint32_t vecsize = 0; MSLShaderVariableRate rate = MSL_SHADER_VARIABLE_RATE_PER_VERTEX; }; @@ -104,7 +105,7 @@ struct MSLShaderInterfaceVariable // become a [[buffer(N)]], [[texture(N)]] or [[sampler(N)]] depending on the resource types used. struct MSLResourceBinding { - spv::ExecutionModel stage = spv::ExecutionModelMax; + ExecutionModel stage = ExecutionModelMax; SPIRType::BaseType basetype = SPIRType::Unknown; uint32_t desc_set = 0; uint32_t binding = 0; @@ -287,9 +288,6 @@ static const uint32_t kArgumentBufferBinding = ~(3u); static const uint32_t kMaxArgumentBuffers = 8; -// The arbitrary maximum for the nesting of array of array copies. -static const uint32_t kArrayCopyMultidimMax = 6; - // Decompiles SPIR-V to Metal Shading Language class CompilerMSL : public CompilerGLSL { @@ -319,6 +317,8 @@ public: uint32_t shader_input_buffer_index = 22; uint32_t shader_index_buffer_index = 21; uint32_t shader_patch_input_buffer_index = 20; + uint32_t draw_id_buffer_index = 19; + uint32_t reversed_depth_viewport_buffer_index = 18; uint32_t shader_input_wg_index = 0; uint32_t device_index = 0; uint32_t enable_frag_output_mask = 0xffffffff; @@ -327,6 +327,8 @@ public: // of the shader with the additional fixed sample mask. uint32_t additional_fixed_sample_mask = 0xffffffff; bool enable_point_size_builtin = true; + bool enable_point_size_default = false; + float default_point_size = 1.0f; bool enable_frag_depth_builtin = true; bool enable_frag_stencil_ref_builtin = true; bool disable_rasterization = false; @@ -338,6 +340,7 @@ public: bool view_index_from_device_index = false; bool dispatch_base = false; bool texture_1D_as_2D = false; + bool emulate_reversed_depth_viewport = false; // Enable use of Metal argument buffers. // MSL 2.0 must also be enabled. @@ -519,6 +522,34 @@ public: // transformed. bool agx_manual_cube_grad_fixup = false; + // Metal will discard fragments with side effects under certain circumstances prematurely. + // Example: CTS test dEQP-VK.fragment_operations.early_fragment.discard_no_early_fragment_tests_depth + // Test will render a full screen quad with varying depth [0,1] for each fragment. + // Each fragment will do an operation with side effects, modify the depth value and + // discard the fragment. The test expects the fragment to be run due to: + // https://registry.khronos.org/vulkan/specs/1.0-extensions/html/vkspec.html#fragops-shader-depthreplacement + // which states that the fragment shader must be run due to replacing the depth in shader. + // However, Metal may prematurely discards fragments without executing them + // (I believe this to be due to a greedy optimization on their end) making the test fail. + // This option enforces fragment execution for such cases where the fragment has operations + // with side effects. Provided as an option hoping Metal will fix this issue in the future. + bool force_fragment_with_side_effects_execution = false; + + // If set, adds a depth pass through statement to circumvent the following issue: + // When the same depth/stencil is used as input and depth/stencil attachment, we need to + // force Metal to perform the depth/stencil write after fragment execution. Otherwise, + // Metal will write to the depth attachment before fragment execution. This happens + // if the fragment does not modify the depth value. + bool input_attachment_is_ds_attachment = false; + + // If BuiltInPosition is not written, automatically disable rasterization. + // The result can be queried with get_is_rasterization_disabled. + bool auto_disable_rasterization = false; + + // Use Fast Math pragmas in MSL code, based on SPIR-V float controls and FP ExecutionModes. + // Requires MSL 3.2 or above, and has no effect with earlier MSL versions. + bool use_fast_math_pragmas = false; + bool is_ios() const { return platform == iOS; @@ -564,9 +595,9 @@ public: // rasterization if vertex shader requires rasterization to be disabled. bool get_is_rasterization_disabled() const { - return is_rasterization_disabled && (get_entry_point().model == spv::ExecutionModelVertex || - get_entry_point().model == spv::ExecutionModelTessellationControl || - get_entry_point().model == spv::ExecutionModelTessellationEvaluation); + return is_rasterization_disabled && (get_entry_point().model == ExecutionModelVertex || + get_entry_point().model == ExecutionModelTessellationControl || + get_entry_point().model == ExecutionModelTessellationEvaluation); } // Provide feedback to calling API to allow it to pass an auxiliary @@ -583,6 +614,14 @@ public: return !buffers_requiring_array_length.empty(); } + // Provide feedback to calling API to determine if the vertex shader writes + // to PointSize. This allows the API to avoid declaring a point size output + // when it is not needed. + bool get_writes_to_point_size() const + { + return writes_to_point_size; + } + bool buffer_requires_array_length(VariableID id) const { return buffers_requiring_array_length.count(id) != 0; @@ -679,20 +718,20 @@ public: // This is typical for tessellation builtin inputs such as tess levels, gl_Position, etc. // This returns k_unknown_location if the location was explicitly assigned with // add_msl_shader_input or the builtin is not used, otherwise returns N in [[attribute(N)]]. - uint32_t get_automatic_builtin_input_location(spv::BuiltIn builtin) const; + uint32_t get_automatic_builtin_input_location(BuiltIn builtin) const; // If not using add_msl_shader_output, it's possible // that certain builtin attributes need to be automatically assigned locations. // This is typical for tessellation builtin outputs such as tess levels, gl_Position, etc. // This returns k_unknown_location if the location were explicitly assigned with // add_msl_shader_output or the builtin were not used, otherwise returns N in [[attribute(N)]]. - uint32_t get_automatic_builtin_output_location(spv::BuiltIn builtin) const; + uint32_t get_automatic_builtin_output_location(BuiltIn builtin) const; // NOTE: Only resources which are remapped using add_msl_resource_binding will be reported here. // Constexpr samplers are always assumed to be emitted. // No specific MSLResourceBinding remapping is required for constexpr samplers as long as they are remapped // by remap_constexpr_sampler(_by_binding). - bool is_msl_resource_binding_used(spv::ExecutionModel model, uint32_t set, uint32_t binding) const; + bool is_msl_resource_binding_used(ExecutionModel model, uint32_t set, uint32_t binding) const; // This must only be called after a successful call to CompilerMSL::compile(). // For a variable resource ID obtained through reflection API, report the automatically assigned resource index. @@ -739,6 +778,19 @@ public: void set_combined_sampler_suffix(const char *suffix); const char *get_combined_sampler_suffix() const; + // Information about specialization constants that are translated into MSL macros + // instead of using function constant + // These must only be called after a successful call to CompilerMSL::compile(). + bool specialization_constant_is_macro(uint32_t constant_id) const; + + // Returns a mask of SPIR-V FP Fast Math Mode flags, that represents the set of flags that can be applied + // across all floating-point types. Each FPFastMathDefault execution mode operation identifies the flags + // for one floating-point type, and the value returned here is a bitwise-AND combination across all types. + // If incl_ops is enabled, the FPFastMathMode of any SPIR-V operations are also included in the bitwise-AND + // to determine the minimal fast-math that applies to all default execution modes and all operations. + // The returned value is also affected by execution modes SignedZeroInfNanPreserve and ContractionOff. + uint32_t get_fp_fast_math_flags(bool incl_ops) const; + protected: // An enum of SPIR-V functions that are implemented in additional // source code that is added to the shader if necessary. @@ -746,21 +798,17 @@ protected: { SPVFuncImplNone, SPVFuncImplMod, + SPVFuncImplSMod, SPVFuncImplRadians, SPVFuncImplDegrees, SPVFuncImplFindILsb, SPVFuncImplFindSMsb, SPVFuncImplFindUMsb, SPVFuncImplSSign, - SPVFuncImplArrayCopyMultidimBase, - // Unfortunately, we cannot use recursive templates in the MSL compiler properly, - // so stamp out variants up to some arbitrary maximum. - SPVFuncImplArrayCopy = SPVFuncImplArrayCopyMultidimBase + 1, - SPVFuncImplArrayOfArrayCopy2Dim = SPVFuncImplArrayCopyMultidimBase + 2, - SPVFuncImplArrayOfArrayCopy3Dim = SPVFuncImplArrayCopyMultidimBase + 3, - SPVFuncImplArrayOfArrayCopy4Dim = SPVFuncImplArrayCopyMultidimBase + 4, - SPVFuncImplArrayOfArrayCopy5Dim = SPVFuncImplArrayCopyMultidimBase + 5, - SPVFuncImplArrayOfArrayCopy6Dim = SPVFuncImplArrayCopyMultidimBase + 6, + SPVFuncImplArrayCopy, + SPVFuncImplArrayCopyMultidim, + SPVFuncImplArrayCopyExtendedSrc, + SPVFuncImplArrayCopyExtendedDst, SPVFuncImplTexelBufferCoords, SPVFuncImplImage2DAtomicCoords, // Emulate texture2D atomic operations SPVFuncImplGradientCube, @@ -774,14 +822,15 @@ protected: SPVFuncImplInverse4x4, SPVFuncImplInverse3x3, SPVFuncImplInverse2x2, - // It is very important that this come before *Swizzle and ChromaReconstruct*, to ensure it's - // emitted before them. - SPVFuncImplForwardArgs, - // Likewise, this must come before *Swizzle. + // It is very important that this come before *Swizzle, to ensure it's emitted before them. SPVFuncImplGetSwizzle, SPVFuncImplTextureSwizzle, + SPVFuncImplGatherReturn, + SPVFuncImplGatherCompareReturn, SPVFuncImplGatherSwizzle, SPVFuncImplGatherCompareSwizzle, + SPVFuncImplGatherConstOffsets, + SPVFuncImplGatherCompareConstOffsets, SPVFuncImplSubgroupBroadcast, SPVFuncImplSubgroupBroadcastFirst, SPVFuncImplSubgroupBallot, @@ -794,6 +843,30 @@ protected: SPVFuncImplSubgroupShuffleXor, SPVFuncImplSubgroupShuffleUp, SPVFuncImplSubgroupShuffleDown, + SPVFuncImplSubgroupRotate, + SPVFuncImplSubgroupClusteredAdd, + SPVFuncImplSubgroupClusteredFAdd = SPVFuncImplSubgroupClusteredAdd, + SPVFuncImplSubgroupClusteredIAdd = SPVFuncImplSubgroupClusteredAdd, + SPVFuncImplSubgroupClusteredMul, + SPVFuncImplSubgroupClusteredFMul = SPVFuncImplSubgroupClusteredMul, + SPVFuncImplSubgroupClusteredIMul = SPVFuncImplSubgroupClusteredMul, + SPVFuncImplSubgroupClusteredMin, + SPVFuncImplSubgroupClusteredFMin = SPVFuncImplSubgroupClusteredMin, + SPVFuncImplSubgroupClusteredSMin = SPVFuncImplSubgroupClusteredMin, + SPVFuncImplSubgroupClusteredUMin = SPVFuncImplSubgroupClusteredMin, + SPVFuncImplSubgroupClusteredMax, + SPVFuncImplSubgroupClusteredFMax = SPVFuncImplSubgroupClusteredMax, + SPVFuncImplSubgroupClusteredSMax = SPVFuncImplSubgroupClusteredMax, + SPVFuncImplSubgroupClusteredUMax = SPVFuncImplSubgroupClusteredMax, + SPVFuncImplSubgroupClusteredAnd, + SPVFuncImplSubgroupClusteredBitwiseAnd = SPVFuncImplSubgroupClusteredAnd, + SPVFuncImplSubgroupClusteredLogicalAnd = SPVFuncImplSubgroupClusteredAnd, + SPVFuncImplSubgroupClusteredOr, + SPVFuncImplSubgroupClusteredBitwiseOr = SPVFuncImplSubgroupClusteredOr, + SPVFuncImplSubgroupClusteredLogicalOr = SPVFuncImplSubgroupClusteredOr, + SPVFuncImplSubgroupClusteredXor, + SPVFuncImplSubgroupClusteredBitwiseXor = SPVFuncImplSubgroupClusteredXor, + SPVFuncImplSubgroupClusteredLogicalXor = SPVFuncImplSubgroupClusteredXor, SPVFuncImplQuadBroadcast, SPVFuncImplQuadSwap, SPVFuncImplReflectScalar, @@ -823,7 +896,14 @@ protected: SPVFuncImplVariableDescriptor, SPVFuncImplVariableSizedDescriptor, SPVFuncImplVariableDescriptorArray, - SPVFuncImplPaddedStd140 + SPVFuncImplPaddedStd140, + SPVFuncImplPaddedArrayElement, + SPVFuncImplReduceAdd, + SPVFuncImplImageFence, + SPVFuncImplTextureCast, + SPVFuncImplMulExtended, + SPVFuncImplSetMeshOutputsEXT, + SPVFuncImplAssume, }; // If the underlying resource has been used for comparison then duplicate loads of that resource must be too @@ -841,6 +921,11 @@ protected: void emit_function_prototype(SPIRFunction &func, const Bitset &return_flags) override; void emit_sampled_image_op(uint32_t result_type, uint32_t result_id, uint32_t image_id, uint32_t samp_id) override; void emit_subgroup_op(const Instruction &i) override; + void emit_subgroup_cluster_op(uint32_t result_type, uint32_t result_id, uint32_t cluster_size, uint32_t op0, + const char *op); + void emit_subgroup_cluster_op_cast(uint32_t result_type, uint32_t result_id, uint32_t cluster_size, uint32_t op0, + const char *op, SPIRType::BaseType input_type, + SPIRType::BaseType expected_result_type); std::string to_texture_op(const Instruction &i, bool sparse, bool *forward, SmallVector &inherited_expressions) override; void emit_fixup() override; @@ -848,23 +933,26 @@ protected: const std::string &qualifier = ""); void emit_struct_member(const SPIRType &type, uint32_t member_type_id, uint32_t index, const std::string &qualifier = "", uint32_t base_offset = 0) override; - void emit_struct_padding_target(const SPIRType &type) override; std::string type_to_glsl(const SPIRType &type, uint32_t id, bool member); std::string type_to_glsl(const SPIRType &type, uint32_t id = 0) override; void emit_block_hints(const SPIRBlock &block) override; + void emit_mesh_entry_point(); + void emit_mesh_outputs(); + void emit_mesh_tasks(SPIRBlock &block) override; + void emit_workgroup_initialization(const SPIRVariable &var) override; // Allow Metal to use the array template to make arrays a value type - std::string type_to_array_glsl(const SPIRType &type) override; + std::string type_to_array_glsl(const SPIRType &type, uint32_t variable_id) override; std::string constant_op_expression(const SPIRConstantOp &cop) override; - bool variable_decl_is_remapped_storage(const SPIRVariable &variable, spv::StorageClass storage) const override; + bool variable_decl_is_remapped_storage(const SPIRVariable &variable, StorageClass storage) const override; // GCC workaround of lambdas calling protected functions (for older GCC versions) std::string variable_decl(const SPIRType &type, const std::string &name, uint32_t id = 0) override; - std::string image_type_glsl(const SPIRType &type, uint32_t id = 0) override; - std::string sampler_type(const SPIRType &type, uint32_t id); - std::string builtin_to_glsl(spv::BuiltIn builtin, spv::StorageClass storage) override; + std::string image_type_glsl(const SPIRType &type, uint32_t id, bool member) override; + std::string sampler_type(const SPIRType &type, uint32_t id, bool member); + std::string builtin_to_glsl(BuiltIn builtin, StorageClass storage) override; std::string to_func_call_arg(const SPIRFunction::Parameter &arg, uint32_t id) override; std::string to_name(uint32_t id, bool allow_alias = true) const override; std::string to_function_name(const TextureFunctionNameArguments &args) override; @@ -876,7 +964,7 @@ protected: bool is_packed, bool row_major) override; // Returns true for BuiltInSampleMask because gl_SampleMask[] is an array in SPIR-V, but [[sample_mask]] is a scalar in Metal. - bool builtin_translates_to_nonarray(spv::BuiltIn builtin) const override; + bool builtin_translates_to_nonarray(BuiltIn builtin) const override; std::string bitcast_glsl_op(const SPIRType &result_type, const SPIRType &argument_type) override; bool emit_complex_bitcast(uint32_t result_id, uint32_t id, uint32_t op0) override; @@ -903,6 +991,7 @@ protected: bool is_tesc_shader() const; bool is_tese_shader() const; + bool is_mesh_shader() const; void preprocess_op_codes(); void localize_global_variables(); @@ -915,8 +1004,9 @@ protected: void extract_global_variables_from_function(uint32_t func_id, std::set &added_arg_ids, std::unordered_set &global_var_ids, std::unordered_set &processed_func_ids); - uint32_t add_interface_block(spv::StorageClass storage, bool patch = false); - uint32_t add_interface_block_pointer(uint32_t ib_var_id, spv::StorageClass storage); + uint32_t add_interface_block(StorageClass storage, bool patch = false); + uint32_t add_interface_block_pointer(uint32_t ib_var_id, StorageClass storage); + uint32_t add_meshlet_block(bool per_primitive); struct InterfaceBlockMeta { @@ -936,37 +1026,39 @@ protected: std::string to_tesc_invocation_id(); void emit_local_masked_variable(const SPIRVariable &masked_var, bool strip_array); - void add_variable_to_interface_block(spv::StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, + void add_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var, InterfaceBlockMeta &meta); - void add_composite_variable_to_interface_block(spv::StorageClass storage, const std::string &ib_var_ref, + void add_composite_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var, InterfaceBlockMeta &meta); - void add_plain_variable_to_interface_block(spv::StorageClass storage, const std::string &ib_var_ref, + void add_plain_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var, InterfaceBlockMeta &meta); - bool add_component_variable_to_interface_block(spv::StorageClass storage, const std::string &ib_var_ref, + bool add_component_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRVariable &var, const SPIRType &type, InterfaceBlockMeta &meta); - void add_plain_member_variable_to_interface_block(spv::StorageClass storage, + void add_plain_member_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var, SPIRType &var_type, uint32_t mbr_idx, InterfaceBlockMeta &meta, const std::string &mbr_name_qual, const std::string &var_chain_qual, uint32_t &location, uint32_t &var_mbr_idx); - void add_composite_member_variable_to_interface_block(spv::StorageClass storage, + void add_composite_member_variable_to_interface_block(StorageClass storage, const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var, SPIRType &var_type, uint32_t mbr_idx, InterfaceBlockMeta &meta, const std::string &mbr_name_qual, const std::string &var_chain_qual, - uint32_t &location, uint32_t &var_mbr_idx); + uint32_t &location, uint32_t &var_mbr_idx, + const Bitset &interpolation_qual); void add_tess_level_input_to_interface_block(const std::string &ib_var_ref, SPIRType &ib_type, SPIRVariable &var); void add_tess_level_input(const std::string &base_ref, const std::string &mbr_name, SPIRVariable &var); - void fix_up_interface_member_indices(spv::StorageClass storage, uint32_t ib_type_id); + void ensure_struct_members_valid_vecsizes(SPIRType &struct_type, uint32_t &location); + void fix_up_interface_member_indices(StorageClass storage, uint32_t ib_type_id); void mark_location_as_used_by_shader(uint32_t location, const SPIRType &type, - spv::StorageClass storage, bool fallback = false); - uint32_t ensure_correct_builtin_type(uint32_t type_id, spv::BuiltIn builtin); + StorageClass storage, bool fallback = false); + uint32_t ensure_correct_builtin_type(uint32_t type_id, BuiltIn builtin); uint32_t ensure_correct_input_type(uint32_t type_id, uint32_t location, uint32_t component, uint32_t num_components, bool strip_array); @@ -981,6 +1073,9 @@ protected: void fix_up_shader_inputs_outputs(); + bool entry_point_is_vertex() const; + bool entry_point_returns_stage_output() const; + bool entry_point_requires_const_device_buffers() const; std::string func_type_decl(SPIRType &type); std::string entry_point_args_classic(bool append_comma); std::string entry_point_args_argument_buffer(bool append_comma); @@ -994,31 +1089,33 @@ protected: std::string to_buffer_size_expression(uint32_t id); bool is_sample_rate() const; bool is_intersection_query() const; - bool is_direct_input_builtin(spv::BuiltIn builtin); - std::string builtin_qualifier(spv::BuiltIn builtin); - std::string builtin_type_decl(spv::BuiltIn builtin, uint32_t id = 0); - std::string built_in_func_arg(spv::BuiltIn builtin, bool prefix_comma); + bool is_direct_input_builtin(BuiltIn builtin); + std::string builtin_qualifier(BuiltIn builtin); + std::string builtin_type_decl(BuiltIn builtin, uint32_t id = 0); + std::string built_in_func_arg(BuiltIn builtin, bool prefix_comma); std::string member_attribute_qualifier(const SPIRType &type, uint32_t index); std::string member_location_attribute_qualifier(const SPIRType &type, uint32_t index); std::string argument_decl(const SPIRFunction::Parameter &arg); - const char *descriptor_address_space(uint32_t id, spv::StorageClass storage, const char *plain_address_space) const; + const char *descriptor_address_space(uint32_t id, StorageClass storage, const char *plain_address_space) const; std::string round_fp_tex_coords(std::string tex_coords, bool coord_is_fp); uint32_t get_metal_resource_index(SPIRVariable &var, SPIRType::BaseType basetype, uint32_t plane = 0); uint32_t get_member_location(uint32_t type_id, uint32_t index, uint32_t *comp = nullptr) const; - uint32_t get_or_allocate_builtin_input_member_location(spv::BuiltIn builtin, + uint32_t get_or_allocate_builtin_input_member_location(BuiltIn builtin, uint32_t type_id, uint32_t index, uint32_t *comp = nullptr); - uint32_t get_or_allocate_builtin_output_member_location(spv::BuiltIn builtin, + uint32_t get_or_allocate_builtin_output_member_location(BuiltIn builtin, uint32_t type_id, uint32_t index, uint32_t *comp = nullptr); - uint32_t get_physical_tess_level_array_size(spv::BuiltIn builtin) const; + uint32_t get_physical_tess_level_array_size(BuiltIn builtin) const; + + uint32_t get_physical_type_id_stride(TypeID type_id) const override; // MSL packing rules. These compute the effective packing rules as observed by the MSL compiler in the MSL output. // These values can change depending on various extended decorations which control packing rules. // We need to make these rules match up with SPIR-V declared rules. - uint32_t get_declared_type_size_msl(const SPIRType &type, bool packed, bool row_major) const; - uint32_t get_declared_type_array_stride_msl(const SPIRType &type, bool packed, bool row_major) const; - uint32_t get_declared_type_matrix_stride_msl(const SPIRType &type, bool packed, bool row_major) const; - uint32_t get_declared_type_alignment_msl(const SPIRType &type, bool packed, bool row_major) const; + uint32_t get_declared_type_size_msl(TypeID type_id, const SPIRType *special_type, bool packed, bool row_major) const; + uint32_t get_declared_type_array_stride_msl(TypeID type_id, const SPIRType *special_type, bool packed, bool row_major) const; + uint32_t get_declared_type_matrix_stride_msl(TypeID type_id, const SPIRType *special_type, bool packed, bool row_major) const; + uint32_t get_declared_type_alignment_msl(TypeID type_id, const SPIRType *special_type, bool packed, bool row_major) const; uint32_t get_declared_struct_member_size_msl(const SPIRType &struct_type, uint32_t index) const; uint32_t get_declared_struct_member_array_stride_msl(const SPIRType &struct_type, uint32_t index) const; @@ -1030,11 +1127,10 @@ protected: uint32_t get_declared_input_matrix_stride_msl(const SPIRType &struct_type, uint32_t index) const; uint32_t get_declared_input_alignment_msl(const SPIRType &struct_type, uint32_t index) const; - const SPIRType &get_physical_member_type(const SPIRType &struct_type, uint32_t index) const; + TypeID get_physical_member_type_id(const SPIRType &struct_type, uint32_t index) const; SPIRType get_presumed_input_type(const SPIRType &struct_type, uint32_t index) const; - uint32_t get_declared_struct_size_msl(const SPIRType &struct_type, bool ignore_alignment = false, - bool ignore_padding = false) const; + uint32_t get_declared_struct_size_msl(const SPIRType &struct_type) const; std::string to_component_argument(uint32_t id); void align_struct(SPIRType &ib_type, std::unordered_set &aligned_structs); @@ -1042,8 +1138,12 @@ protected: void mark_struct_members_packed(const SPIRType &type); void ensure_member_packing_rules_msl(SPIRType &ib_type, uint32_t index); bool validate_member_packing_rules_msl(const SPIRType &type, uint32_t index) const; - std::string get_argument_address_space(const SPIRVariable &argument); + std::string get_variable_address_space(const SPIRVariable &argument); + // Special case of get_variable_address_space which is only used for leaf functions. + std::string get_leaf_argument_address_space(const SPIRVariable &argument); std::string get_type_address_space(const SPIRType &type, uint32_t id, bool argument = false); + bool decoration_flags_signal_volatile(const Bitset &flags) const; + bool decoration_flags_signal_coherent(const Bitset &flags) const; const char *to_restrict(uint32_t id, bool space); SPIRType &get_stage_in_struct_type(); SPIRType &get_stage_out_struct_type(); @@ -1052,15 +1152,15 @@ protected: std::string get_tess_factor_struct_name(); SPIRType &get_uint_type(); uint32_t get_uint_type_id(); - void emit_atomic_func_op(uint32_t result_type, uint32_t result_id, const char *op, spv::Op opcode, + void emit_atomic_func_op(uint32_t result_type, uint32_t result_id, const char *op, Op opcode, uint32_t mem_order_1, uint32_t mem_order_2, bool has_mem_order_2, uint32_t op0, uint32_t op1 = 0, bool op1_is_pointer = false, bool op1_is_literal = false, uint32_t op2 = 0); const char *get_memory_order(uint32_t spv_mem_sem); - void add_pragma_line(const std::string &line); + void add_pragma_line(const std::string &line, bool recompile_on_unique); void add_typedef_line(const std::string &line); void emit_barrier(uint32_t id_exe_scope, uint32_t id_mem_scope, uint32_t id_mem_sem); bool emit_array_copy(const char *expr, uint32_t lhs_id, uint32_t rhs_id, - spv::StorageClass lhs_storage, spv::StorageClass rhs_storage) override; + StorageClass lhs_storage, StorageClass rhs_storage) override; void build_implicit_builtins(); uint32_t build_constant_uint_array_pointer(); void emit_entry_point_declarations() override; @@ -1084,11 +1184,18 @@ protected: uint32_t builtin_stage_input_size_id = 0; uint32_t builtin_local_invocation_index_id = 0; uint32_t builtin_workgroup_size_id = 0; + uint32_t builtin_mesh_primitive_indices_id = 0; + uint32_t builtin_mesh_sizes_id = 0; + uint32_t builtin_task_grid_id = 0; + uint32_t builtin_frag_depth_id = 0; uint32_t swizzle_buffer_id = 0; uint32_t buffer_size_buffer_id = 0; uint32_t view_mask_buffer_id = 0; + uint32_t draw_index_buffer_id = 0; uint32_t dynamic_offsets_buffer_id = 0; uint32_t uint_type_id = 0; + uint32_t shared_uint_type_id = 0; + uint32_t meshlet_type_id = 0; uint32_t argument_buffer_padding_buffer_type_id = 0; uint32_t argument_buffer_padding_image_type_id = 0; uint32_t argument_buffer_padding_sampler_type_id = 0; @@ -1101,20 +1208,21 @@ protected: void emit_store_statement(uint32_t lhs_expression, uint32_t rhs_expression) override; void analyze_sampled_image_usage(); + void analyze_workgroup_variables(); bool access_chain_needs_stage_io_builtin_translation(uint32_t base) override; - bool prepare_access_chain_for_scalar_access(std::string &expr, const SPIRType &type, spv::StorageClass storage, + bool prepare_access_chain_for_scalar_access(std::string &expr, const SPIRType &type, StorageClass storage, bool &is_packed) override; void fix_up_interpolant_access_chain(const uint32_t *ops, uint32_t length); - void check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type) override; + bool check_physical_type_cast(std::string &expr, const SPIRType *type, uint32_t physical_type) override; bool emit_tessellation_access_chain(const uint32_t *ops, uint32_t length); bool emit_tessellation_io_load(uint32_t result_type, uint32_t id, uint32_t ptr); bool is_out_of_bounds_tessellation_level(uint32_t id_lhs); - void ensure_builtin(spv::StorageClass storage, spv::BuiltIn builtin); + void ensure_builtin(StorageClass storage, BuiltIn builtin); - void mark_implicit_builtin(spv::StorageClass storage, spv::BuiltIn builtin, uint32_t id); + void mark_implicit_builtin(StorageClass storage, BuiltIn builtin, uint32_t id); std::string convert_to_f32(const std::string &expr, uint32_t components); @@ -1132,9 +1240,10 @@ protected: std::unordered_map fragment_output_components; std::unordered_map builtin_to_automatic_input_location; std::unordered_map builtin_to_automatic_output_location; - std::set pragma_lines; - std::set typedef_lines; + std::vector pragma_lines; + std::vector typedef_lines; SmallVector vars_needing_early_declaration; + std::unordered_set constant_macro_ids; std::unordered_map, InternalHasher> resource_bindings; std::unordered_map resource_arg_buff_idx_to_binding_number; @@ -1153,6 +1262,8 @@ protected: VariableID stage_out_ptr_var_id = 0; VariableID tess_level_inner_var_id = 0; VariableID tess_level_outer_var_id = 0; + VariableID mesh_out_per_vertex = 0; + VariableID mesh_out_per_primitive = 0; VariableID stage_out_masked_builtin_type_id = 0; // Handle HLSL-style 0-based vertex/instance index. @@ -1172,15 +1283,22 @@ protected: bool using_builtin_array() const; bool is_rasterization_disabled = false; + bool has_descriptor_side_effects_buffer = false; bool capture_output_to_buffer = false; bool needs_swizzle_buffer_def = false; bool used_swizzle_buffer = false; bool added_builtin_tess_level = false; + bool needs_local_invocation_index = false; bool needs_subgroup_invocation_id = false; bool needs_subgroup_size = false; bool needs_sample_id = false; bool needs_helper_invocation = false; + bool needs_workgroup_zero_init = false; + bool needs_point_size_output = false; + bool writes_to_depth = false; + bool writes_to_point_size = false; std::string qual_pos_var_name; + std::string qual_viewport_idx_var_name; std::string stage_in_var_name = "in"; std::string stage_out_var_name = "out"; std::string patch_stage_in_var_name = "patchIn"; @@ -1196,7 +1314,7 @@ protected: std::string patch_output_buffer_var_name = "spvPatchOut"; std::string tess_factor_buffer_var_name = "spvTessLevel"; std::string index_buffer_var_name = "spvIndices"; - spv::Op previous_instruction_opcode = spv::OpNop; + Op previous_instruction_opcode = OpNop; // Must be ordered since declaration is in a specific order. std::map constexpr_samplers_by_id; @@ -1204,7 +1322,6 @@ protected: const MSLConstexprSampler *find_constexpr_sampler(uint32_t id) const; std::unordered_set buffers_requiring_array_length; - SmallVector> buffer_aliases_argument; SmallVector buffer_aliases_discrete; std::unordered_set atomic_image_vars_emulated; // Emulate texture2D atomic operations std::unordered_set pull_model_inputs; @@ -1213,7 +1330,13 @@ protected: SmallVector entry_point_bindings; // Must be ordered since array is in a specific order. - std::map> buffers_requiring_dynamic_offset; + struct DynamicBuffer + { + uint32_t base_index; + uint32_t var_id; + std::string mbr_name; + }; + std::map buffers_requiring_dynamic_offset; SmallVector disabled_frag_outputs; @@ -1223,9 +1346,12 @@ protected: uint32_t argument_buffer_discrete_mask = 0; uint32_t argument_buffer_device_storage_mask = 0; + void emit_argument_buffer_aliased_descriptor(const SPIRVariable &aliased_var, + const SPIRVariable &base_var); + void analyze_argument_buffers(); bool descriptor_set_is_argument_buffer(uint32_t desc_set) const; - MSLResourceBinding &get_argument_buffer_resource(uint32_t desc_set, uint32_t arg_idx); + const MSLResourceBinding &get_argument_buffer_resource(uint32_t desc_set, uint32_t arg_idx) const; void add_argument_buffer_padding_buffer_type(SPIRType &struct_type, uint32_t &mbr_idx, uint32_t &arg_buff_index, MSLResourceBinding &rez_bind); void add_argument_buffer_padding_image_type(SPIRType &struct_type, uint32_t &mbr_idx, uint32_t &arg_buff_index, MSLResourceBinding &rez_bind); void add_argument_buffer_padding_sampler_type(SPIRType &struct_type, uint32_t &mbr_idx, uint32_t &arg_buff_index, MSLResourceBinding &rez_bind); @@ -1237,17 +1363,17 @@ protected: uint32_t build_msl_interpolant_type(uint32_t type_id, bool is_noperspective); bool suppress_missing_prototypes = false; + bool suppress_incompatible_pointer_types_discard_qualifiers = false; + bool suppress_sometimes_unitialized = false; void add_spv_func_and_recompile(SPVFuncImpl spv_func); void activate_argument_buffer_resources(); bool type_is_msl_framebuffer_fetch(const SPIRType &type) const; - bool type_is_pointer(const SPIRType &type) const; - bool type_is_pointer_to_pointer(const SPIRType &type) const; bool is_supported_argument_buffer_type(const SPIRType &type) const; - bool variable_storage_requires_stage_io(spv::StorageClass storage) const; + bool variable_storage_requires_stage_io(StorageClass storage) const; bool needs_manual_helper_invocation_updates() const { @@ -1255,7 +1381,7 @@ protected: } bool needs_frag_discard_checks() const { - return get_execution_model() == spv::ExecutionModelFragment && msl_options.supports_msl_version(2, 3) && + return get_execution_model() == ExecutionModelFragment && msl_options.supports_msl_version(2, 3) && msl_options.check_discarded_frag_stores && frag_shader_needs_discard_checks; } @@ -1265,40 +1391,41 @@ protected: // OpcodeHandler that handles several MSL preprocessing operations. struct OpCodePreprocessor : OpcodeHandler { - OpCodePreprocessor(CompilerMSL &compiler_) - : compiler(compiler_) + explicit OpCodePreprocessor(CompilerMSL &compiler_) + : OpcodeHandler(compiler_), self(compiler_) { + enable_result_types = true; } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t length) override; - CompilerMSL::SPVFuncImpl get_spv_func_impl(spv::Op opcode, const uint32_t *args); + bool handle(Op opcode, const uint32_t *args, uint32_t length) override; + CompilerMSL::SPVFuncImpl get_spv_func_impl(Op opcode, const uint32_t *args, uint32_t length); void check_resource_write(uint32_t var_id); - CompilerMSL &compiler; - std::unordered_map result_types; + CompilerMSL &self; std::unordered_map image_pointers_emulated; // Emulate texture2D atomic operations bool suppress_missing_prototypes = false; bool uses_atomics = false; bool uses_image_write = false; bool uses_buffer_write = false; bool uses_discard = false; + bool needs_local_invocation_index = false; bool needs_subgroup_invocation_id = false; bool needs_subgroup_size = false; bool needs_sample_id = false; bool needs_helper_invocation = false; + bool uses_cooperative_matrix = false; }; // OpcodeHandler that scans for uses of sampled images struct SampledImageScanner : OpcodeHandler { - SampledImageScanner(CompilerMSL &compiler_) - : compiler(compiler_) + explicit SampledImageScanner(CompilerMSL &compiler_) + : OpcodeHandler(compiler_), self(compiler_) { } - bool handle(spv::Op opcode, const uint32_t *args, uint32_t) override; - - CompilerMSL &compiler; + CompilerMSL &self; + bool handle(Op opcode, const uint32_t *args, uint32_t) override; }; // Sorts the members of a SPIRType and associated Meta info based on a settable sorting diff --git a/src/libraries/spirv_cross/spirv_parser.cpp b/src/libraries/spirv_cross/spirv_parser.cpp index 6108dbb65..2cf232a5f 100644 --- a/src/libraries/spirv_cross/spirv_parser.cpp +++ b/src/libraries/spirv_cross/spirv_parser.cpp @@ -22,10 +22,11 @@ */ #include "spirv_parser.hpp" +#include "NonSemanticShaderDebugInfo100.h" #include using namespace std; -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; namespace SPIRV_CROSS_NAMESPACE { @@ -43,7 +44,7 @@ static bool decoration_is_string(Decoration decoration) { switch (decoration) { - case DecorationHlslSemanticGOOGLE: + case DecorationUserSemantic: return true; default: @@ -137,12 +138,42 @@ void Parser::parse() } forward_pointer_fixups.clear(); + for (auto &source : ir.sources) + { + auto cmp = [](const ParsedIR::Source::Marker &a, const ParsedIR::Source::Marker &b) { + return a.line < b.line; + }; + + std::sort(source.line_markers.begin(), source.line_markers.end(), cmp); + } + if (current_function) SPIRV_CROSS_THROW("Function was not terminated."); if (current_block) SPIRV_CROSS_THROW("Block was not terminated."); + + // Now that all definitions are bound to a kind, we can filter the library + // exports and populate the exported functions. + for (uint32_t id : ir.library_exports) + { + if (ir.ids[id].get_type() == TypeFunction) + ir.library_exported_functions.push_back(id); + } + if (ir.default_entry_point == 0) - SPIRV_CROSS_THROW("There is no entry point in the SPIR-V module."); + { + if (ir.library_exported_functions.empty()) + SPIRV_CROSS_THROW("There is no entry point in the SPIR-V module."); + + // No OpEntryPoint, but the module exports functions. Treat as a library + // module: designate the first exported function as the default entry + // point so analyses keyed on default_entry_point can run. + ir.is_library_module = true; + ir.default_entry_point = ir.library_exported_functions.front(); + auto &name = ir.get_name(ir.default_entry_point); + ir.entry_points.insert(std::make_pair(ir.default_entry_point, + SPIREntryPoint(ir.default_entry_point, ExecutionModelGLCompute, name))); + } } const uint32_t *Parser::stream(const Instruction &instr) const @@ -194,7 +225,6 @@ void Parser::parse(const Instruction &instruction) switch (op) { - case OpSourceContinued: case OpSourceExtension: case OpNop: case OpModuleProcessed: @@ -213,38 +243,53 @@ void Parser::parse(const Instruction &instruction) case OpSource: { - auto lang = static_cast(ops[0]); - switch (lang) + ir.sources.emplace_back(); + auto &source = ir.sources.back(); + source.lang = static_cast(ops[0]); + + switch (source.lang) { case SourceLanguageESSL: - ir.source.es = true; - ir.source.version = ops[1]; - ir.source.known = true; - ir.source.hlsl = false; + source.es = true; + source.version = ops[1]; + source.known = true; + source.hlsl = false; break; case SourceLanguageGLSL: - ir.source.es = false; - ir.source.version = ops[1]; - ir.source.known = true; - ir.source.hlsl = false; + source.es = false; + source.version = ops[1]; + source.known = true; + source.hlsl = false; break; case SourceLanguageHLSL: // For purposes of cross-compiling, this is GLSL 450. - ir.source.es = false; - ir.source.version = 450; - ir.source.known = true; - ir.source.hlsl = true; + source.es = false; + source.version = 450; + source.known = true; + source.hlsl = true; break; default: - ir.source.known = false; + source.known = false; break; } + + if (length >= 3) + source.file_id = ops[2]; + + if (length >= 4) + source.source = extract_string(ir.spirv, instruction.offset + 3); + break; } + case OpSourceContinued: + if (!ir.sources.empty()) + ir.sources.back().source += extract_string(ir.spirv, instruction.offset); + break; + case OpUndef: { uint32_t result_type = ops[0]; @@ -305,6 +350,7 @@ void Parser::parse(const Instruction &instruction) } case OpExtInst: + case OpExtInstWithForwardRefsKHR: { // The SPIR-V debug information extended instructions might come at global scope. if (current_block) @@ -317,6 +363,72 @@ void Parser::parse(const Instruction &instruction) ir.load_type_width.insert({ ops[1], type->width }); } } + + if (op == OpExtInst && length > 4) + { + // Don't want to deal with ForwardRefs here. + auto &ext = get(ops[2]); + if (ext.ext == SPIRExtension::NonSemanticShaderDebugInfo) + { + const auto instr = ops[3]; + if (instr == NonSemanticShaderDebugInfo100DebugSource) + { + set(ops[1], get(ops[4]).str); + + ir.sources.emplace_back(); + auto &source = ir.sources.back(); + source.file_id = ops[4]; + source.define_id = ops[1]; + if (length >= 6) + source.source = ir.get(ops[5]).str; + } + else if (instr == NonSemanticShaderDebugInfo100DebugSourceContinued) + { + if (length < 5) + SPIRV_CROSS_THROW("Invalid arguments for ShaderDebugInfo100DebugSourceContinued"); + if (!ir.sources.empty()) + ir.sources.back().source += ir.get(ops[4]).str; + } + else if (instr == NonSemanticShaderDebugInfo100DebugLine) + { + if (length < 9) + SPIRV_CROSS_THROW("Invalid arguments for ShaderDebugInfo100DebugLine"); + auto source_id = ops[4]; + auto line_start = ir.get(ops[5]).scalar_i32(); + auto col_start = ir.get(ops[7]).scalar_i32(); + + for (auto &source : ir.sources) + { + if (source.define_id != source_id) + continue; + + source.line_markers.emplace_back(); + auto &marker = source.line_markers.back(); + marker.line = line_start; + marker.col = col_start; + marker.offset = instruction.offset - 1; + marker.function_id = current_function ? current_function->self : ID(0); + marker.block_id = current_block ? current_block->self : ID(0); + break; + } + } + else if (instr == NonSemanticShaderDebugInfo100DebugLocalVariable) + { + if (length < 11) + SPIRV_CROSS_THROW("Invalid arguments for ShaderDebugInfo100DebugLocalVariable"); + auto &lvar = set(ops[1]); + lvar.name_id = ops[4]; + } + else if (instr == NonSemanticShaderDebugInfo100DebugDeclare) + { + if (length < 7) + SPIRV_CROSS_THROW("Invalid arguments for ShaderDebugInfo100DebugDeclare"); + auto &lvar = get(ops[4]); + auto &var = get(ops[5]); + var.debug_local_variables.push_back(lvar.self); + } + } + } break; } @@ -368,6 +480,30 @@ void Parser::parse(const Instruction &instruction) execution.output_primitives = ops[2]; break; + case ExecutionModeSignedZeroInfNanPreserve: + switch (ops[2]) + { + case 8: + execution.signed_zero_inf_nan_preserve_8 = true; + break; + + case 16: + execution.signed_zero_inf_nan_preserve_16 = true; + break; + + case 32: + execution.signed_zero_inf_nan_preserve_32 = true; + break; + + case 64: + execution.signed_zero_inf_nan_preserve_64 = true; + break; + + default: + SPIRV_CROSS_THROW("Invalid bit-width for SignedZeroInfNanPreserve."); + } + break; + default: break; } @@ -380,13 +516,21 @@ void Parser::parse(const Instruction &instruction) auto mode = static_cast(ops[1]); execution.flags.set(mode); - if (mode == ExecutionModeLocalSizeId) + switch (mode) { + case ExecutionModeLocalSizeId: execution.workgroup_size.id_x = ops[2]; execution.workgroup_size.id_y = ops[3]; execution.workgroup_size.id_z = ops[4]; - } + break; + case ExecutionModeFPFastMathDefault: + execution.fp_fast_math_defaults[ops[2]] = ops[3]; + break; + + default: + break; + } break; } @@ -481,6 +625,19 @@ void Parser::parse(const Instruction &instruction) else ir.set_decoration(id, decoration); + // Track exported functions so we can compile library modules that have no OpEntryPoint. + // LinkageAttributes layout: literal-string (variable words) followed by LinkageType. + if (decoration == DecorationLinkageAttributes && length >= 4 && + static_cast(ops[length - 1]) == LinkageTypeExport) + { + ir.library_exports.push_back(id); + + // If OpName was stripped (e.g. by spirv-opt --strip-debug), fall back + // to the linkage name so the emitted function keeps its export name. + if (ir.get_name(id).empty()) + ir.set_name(id, extract_string(ir.spirv, instruction.offset + 2)); + } + break; } @@ -504,6 +661,12 @@ void Parser::parse(const Instruction &instruction) break; } + // MemberDecorateIdEXT only applies to OffsetIdEXT when descriptors are packed in structs. + // This is currently unsupported and will fail in compilation. + // Pass it through in case someone just needs reflection. + case OpMemberDecorateIdEXT: + break; + case OpMemberDecorateStringGOOGLE: { uint32_t id = ops[0]; @@ -536,12 +699,37 @@ void Parser::parse(const Instruction &instruction) uint32_t id = ops[0]; uint32_t width = ops[1]; auto &type = set(id, op); + + if (width != 16 && width != 8 && length > 2) + SPIRV_CROSS_THROW("Unrecognized FP encoding mode for OpTypeFloat."); + if (width == 64) type.basetype = SPIRType::Double; else if (width == 32) type.basetype = SPIRType::Float; else if (width == 16) - type.basetype = SPIRType::Half; + { + if (length > 2) + { + if (ops[2] == FPEncodingBFloat16KHR) + type.basetype = SPIRType::BFloat16; + else + SPIRV_CROSS_THROW("Unrecognized encoding for OpTypeFloat 16."); + } + else + type.basetype = SPIRType::Half; + } + else if (width == 8) + { + if (length < 2) + SPIRV_CROSS_THROW("Missing encoding for OpTypeFloat 8."); + else if (ops[2] == FPEncodingFloat8E4M3EXT) + type.basetype = SPIRType::FloatE4M3; + else if (ops[2] == FPEncodingFloat8E5M2EXT) + type.basetype = SPIRType::FloatE5M2; + else + SPIRV_CROSS_THROW("Invalid encoding for OpTypeFloat 8."); + } else SPIRV_CROSS_THROW("Unrecognized bit-width of floating point type."); type.width = width; @@ -592,6 +780,40 @@ void Parser::parse(const Instruction &instruction) break; } + case OpTypeCooperativeMatrixKHR: + { + uint32_t id = ops[0]; + auto &base = get(ops[1]); + auto &matrixbase = set(id, base); + + matrixbase.op = op; + matrixbase.ext.cooperative.scope_id = ops[2]; + matrixbase.ext.cooperative.rows_id = ops[3]; + matrixbase.ext.cooperative.columns_id = ops[4]; + matrixbase.ext.cooperative.use_id = ops[5]; + matrixbase.self = id; + matrixbase.parent_type = ops[1]; + break; + } + + case OpTypeCooperativeVectorNV: + { + uint32_t id = ops[0]; + auto &type = set(id, op); + + type.basetype = SPIRType::CoopVecNV; + type.op = op; + type.ext.coopVecNV.component_type_id = ops[1]; + type.ext.coopVecNV.component_count_id = ops[2]; + type.parent_type = ops[1]; + + // CoopVec-Nv can be used with integer operations like SMax where + // where spirv-opt does explicit checks on integer bitwidth + auto component_type = get(type.ext.coopVecNV.component_type_id); + type.width = component_type.width; + break; + } + case OpTypeArray: { uint32_t id = ops[0]; @@ -677,6 +899,7 @@ void Parser::parse(const Instruction &instruction) break; } + case OpTypeUntypedPointerKHR: case OpTypePointer: { uint32_t id = ops[0]; @@ -684,7 +907,7 @@ void Parser::parse(const Instruction &instruction) // Very rarely, we might receive a FunctionPrototype here. // We won't be able to compile it, but we shouldn't crash when parsing. // We should be able to reflect. - auto *base = maybe_get(ops[2]); + auto *base = op == OpTypePointer ? maybe_get(ops[2]) : nullptr; auto &ptrbase = set(id, op); if (base) @@ -703,7 +926,10 @@ void Parser::parse(const Instruction &instruction) if (base && base->forward_pointer) forward_pointer_fixups.push_back({ id, ops[2] }); - ptrbase.parent_type = ops[2]; + if (op == OpTypePointer) + ptrbase.parent_type = ops[2]; + else + ptrbase.basetype = SPIRType::Void; // Do NOT set ptrbase.self! break; @@ -789,6 +1015,20 @@ void Parser::parse(const Instruction &instruction) break; } + case OpTypeTensorARM: + { + uint32_t id = ops[0]; + auto &type = set(id, op); + type.basetype = SPIRType::Tensor; + type.ext.tensor = {}; + type.ext.tensor.type = ops[1]; + if (length >= 3) + type.ext.tensor.rank = ops[2]; + if (length >= 4) + type.ext.tensor.shape = ops[3]; + break; + } + // Variable declaration // All variables are essentially pointers with a storage qualifier. case OpVariable: @@ -809,6 +1049,27 @@ void Parser::parse(const Instruction &instruction) break; } + case OpUntypedVariableKHR: + { + uint32_t type = ops[0]; + uint32_t id = ops[1]; + auto storage = static_cast(ops[2]); + uint32_t data_type = length >= 4 ? ops[3] : 0; + uint32_t initializer = length >= 5 ? ops[4] : 0; + + if (storage == StorageClassFunction) + { + if (!current_function) + SPIRV_CROSS_THROW("No function currently in scope"); + current_function->add_local_variable(id); + } + + auto &v = set(id, type, storage, initializer); + v.untyped = true; + v.untyped_alloca_type = data_type; + break; + } + // OpPhi // OpPhi is a fairly magical opcode. // It selects temporary variables based on which parent block we *came from*. @@ -825,7 +1086,7 @@ void Parser::parse(const Instruction &instruction) uint32_t id = ops[1]; // Instead of a temporary, create a new function-wide temporary with this ID instead. - auto &var = set(id, result_type, spv::StorageClassFunction); + auto &var = set(id, result_type, StorageClassFunction); var.phi_variable = true; current_function->add_local_variable(id); @@ -835,17 +1096,27 @@ void Parser::parse(const Instruction &instruction) break; } - // Constants + // Constants case OpSpecConstant: case OpConstant: + case OpConstantCompositeReplicateEXT: + case OpSpecConstantCompositeReplicateEXT: { uint32_t id = ops[1]; auto &type = get(ops[0]); - - if (type.width > 32) - set(id, ops[0], ops[2] | (uint64_t(ops[3]) << 32), op == OpSpecConstant); + if (op == OpConstantCompositeReplicateEXT || op == OpSpecConstantCompositeReplicateEXT) + { + auto subconstant = uint32_t(ops[2]); + set(id, ops[0], &subconstant, 1, op == OpSpecConstantCompositeReplicateEXT, true); + } else - set(id, ops[0], ops[2], op == OpSpecConstant); + { + + if (type.width > 32) + set(id, ops[0], ops[2] | (uint64_t(ops[3]) << 32), op == OpSpecConstant); + else + set(id, ops[0], ops[2], op == OpSpecConstant); + } break; } @@ -929,6 +1200,24 @@ void Parser::parse(const Instruction &instruction) break; } + case OpConstantSizeOfEXT: + { + uint32_t id = ops[1]; + uint32_t type = ops[0]; + auto &c = set(id, type); + c.size_of_type = ops[2]; + break; + } + + case OpTypeBufferEXT: + { + uint32_t type = ops[0]; + auto &t = set(type, OpTypeBufferEXT); + t.basetype = SPIRType::DescriptorHeapBuffer; + t.ext.descriptor_heap_buffer.storage = static_cast(ops[1]); + break; + } + // Functions case OpFunction: { @@ -1249,6 +1538,24 @@ void Parser::parse(const Instruction &instruction) current_function->entry_line.line_literal = ops[1]; } } + + uint32_t file = ops[0]; + uint32_t line = ops[1]; + + for (auto &source : ir.sources) + { + if (source.file_id == file) + { + source.line_markers.emplace_back(); + auto &marker = source.line_markers.back(); + marker.line = line; + marker.offset = instruction.offset - 1; + marker.function_id = current_function ? current_function->self : ID(0); + marker.block_id = current_block ? current_block->self : ID(0); + break; + } + } + break; } diff --git a/src/libraries/spirv_cross/spirv_reflect.cpp b/src/libraries/spirv_cross/spirv_reflect.cpp index b02773722..380761304 100644 --- a/src/libraries/spirv_cross/spirv_reflect.cpp +++ b/src/libraries/spirv_cross/spirv_reflect.cpp @@ -25,7 +25,7 @@ #include "spirv_glsl.hpp" #include -using namespace spv; +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; using namespace SPIRV_CROSS_NAMESPACE; using namespace std; @@ -449,7 +449,7 @@ void CompilerReflection::emit_type_member_qualifiers(const SPIRType &type, uint3 } } -string CompilerReflection::execution_model_to_str(spv::ExecutionModel model) +string CompilerReflection::execution_model_to_str(ExecutionModel model) { switch (model) { @@ -477,6 +477,12 @@ string CompilerReflection::execution_model_to_str(spv::ExecutionModel model) return "rmiss"; case ExecutionModelCallableNV: return "rcall"; + case ExecutionModelMeshNV: + case ExecutionModelMeshEXT: + return "mesh"; + case ExecutionModelTaskNV: + case ExecutionModelTaskEXT: + return "task"; default: return "???"; } @@ -504,7 +510,9 @@ void CompilerReflection::emit_entry_points() json_stream->begin_json_object(); json_stream->emit_json_key_value("name", e.name); json_stream->emit_json_key_value("mode", execution_model_to_str(e.execution_model)); - if (e.execution_model == ExecutionModelGLCompute) + if (e.execution_model == ExecutionModelGLCompute || e.execution_model == ExecutionModelMeshEXT || + e.execution_model == ExecutionModelMeshNV || e.execution_model == ExecutionModelTaskEXT || + e.execution_model == ExecutionModelTaskNV) { const auto &spv_entry = get_entry_point(e.name, e.execution_model); @@ -547,6 +555,7 @@ void CompilerReflection::emit_resources() emit_resources("push_constants", res.push_constant_buffers); emit_resources("counters", res.atomic_counters); emit_resources("acceleration_structures", res.acceleration_structures); + emit_resources("tensors", res.tensors); } void CompilerReflection::emit_resources(const char *tag, const SmallVector &resources) @@ -633,6 +642,12 @@ void CompilerReflection::emit_resources(const char *tag, const SmallVectoremit_json_key_value("offset", get_decoration(res.id, DecorationOffset)); + if (mask.get(DecorationWeightTextureQCOM)) + json_stream->emit_json_key_value("WeightTextureQCOM", get_decoration(res.id, DecorationWeightTextureQCOM)); + if (mask.get(DecorationBlockMatchTextureQCOM)) + json_stream->emit_json_key_value("BlockMatchTextureQCOM", get_decoration(res.id, DecorationBlockMatchTextureQCOM)); + if (mask.get(DecorationBlockMatchSamplerQCOM)) + json_stream->emit_json_key_value("BlockMatchSamplerQCOM", get_decoration(res.id, DecorationBlockMatchSamplerQCOM)); // For images, the type itself adds a layout qualifer. // Only emit the format for storage images. diff --git a/src/libraries/spirv_cross/spirv_reflect.hpp b/src/libraries/spirv_cross/spirv_reflect.hpp index a129ba54d..492a951f9 100644 --- a/src/libraries/spirv_cross/spirv_reflect.hpp +++ b/src/libraries/spirv_cross/spirv_reflect.hpp @@ -34,6 +34,7 @@ class Stream; namespace SPIRV_CROSS_NAMESPACE { +using namespace SPIRV_CROSS_SPV_HEADER_NAMESPACE; class CompilerReflection : public CompilerGLSL { using Parent = CompilerGLSL; @@ -67,7 +68,7 @@ public: std::string compile() override; private: - static std::string execution_model_to_str(spv::ExecutionModel model); + static std::string execution_model_to_str(ExecutionModel model); void emit_entry_points(); void emit_types();