3dsmax 2010 ssao in .fx

does anyone know if new macros exist (like SHADOW_FUNCTOR) for accessing soft shadows or SSAO data?

I was interested in this so i had a poke around in max’s memory, so this is part of the contents of “shadowmap.fxh” the syntax may not be entirely correct as it was a quick rough out to see what was going on. There were also some missing and spurious character from the memory dump. There’s also a third macro define called AREA_SHADOW_FUNCTOR…

this is the definition for the SHADOW_FUNCTOR


	#ifndef SHADOW_FUNCTOR

	#define SHADOW_FUNC_BODY(shadowTerm)
	float shadowTerm(float4 wPos) 
	{
		if (__IsShadowMapValid_##shadowTerm##)
		{
			float4 lPos = mul(wPos, __ShadowMatrix_##shadowTerm##);
			return ShadowTerm30(lPos,__ShadowMapSize_##shadowTerm##,__DepthSamp_##shadowTerm##, __TransSamp_##shadowTerm##,false, __IsTiledCube_##shadowTerm##,__IndirectCubeSamp_##shadowTerm##,__FaceDataSamp0_##shadowTerm##,__FaceDataSamp1_##shadowTerm##,__FaceDataMapOffset_##shadowTerm##);
		}
		else return 1.0f;
	}
	
	#if NVDST_SUPPORTED.
		#define __FILTER  LINEAR
	#else
		#define __FILTER  POINT
	#endif
	
	#define SHADOW_PARAM(shadowTerm) 
	float4 __PlaceHolder_255_##shadowTerm## : ShadowDataHE 
	<
		string UIName = "fake"; 
		string Object = "PointLight";
		string Space = "World";
		int refID = 0;
	>;
	
	float4x4 __ShadowMatrix_##shadowTerm##;
	float4  __ShadowMapSize_##shadowTerm##;
	texture __DepthTex_##shadowTerm##;
	texture __TransTex_##shadowTerm##;
	textureCUBE __IndirectCubeTex_##shadowTerm##;
	texture2D __FaceDataTex0_##shadowTerm##;
	texture2D __FaceDataTex1_##shadowTerm##;
	bool __IsTrans_##shadowTerm##;
	bool __IsTiledCube_##shadowTerm##;
	bool __IsShadowMapValid_##shadowTerm##;
	float4 __FaceDataMapOffset_##shadowTerm##;
	
	sampler _DepthSamp_##shadowTerm## = sampler_state
	{
		Texture = <__DepthTex_##shadowTerm##>;
		MinFilter = ##__FILTER##; 
		MagFilter = ##__FILTER##; 
		MipFilter = NONE;
		ADDRESSU = CLAMP;
		ADDRESSV = CLAMP; 
	};
	
	sampler  __TransSamp_##shadowTerm## = sampler_state
	{
		Texture= <__TransTex_##shadowTerm##>;
		MinFilter = LINEAR; 
		MagFilter = LINEAR; 
		MipFilter = NONE;
		ADDRESSU = CLAMP;
		ADDRESSV = CLAMP;
	};
	
	samplerCUBE __IndirectCubeSamp_##shadowTerm##= sampler_state{Texture= <__IndirectCubeTex_##shadowTerm##>;
	MinFilter = POINT; 
	MagFilter = POINT; 
	MipFilter = NONE;};
	
	sampler2D __FaceDataSamp0_##shadowTerm##= sampler_state{Texture =<__FaceDataTex0_##shadowTerm##>;
	MinFilter = POINT; 
	MagFilter = POINT; 
	MipFilter = NONE;};
	
	sampler2D __FaceDataSamp1_##shadowTerm##= sampler_state{Texture = <__FaceDataTex1_##shadowTerm##>;
	MinFilter = POINT; 
	MagFilter = POINT; 
	MipFilter = NONE;};
	
	
	#define SHADOW_FUNCTOR(functor)\
			SHADOW_PARAM(functor);\
			SHADOW_FUNC_BODY(functor);
	#endif

And this is the definition for the SOFT_SHADOW_FUNCTOR, it takes Three arguments but I can’t get it to work, maybe someone else can have more luck.


	#ifndef SOFT_SHADOW_FUNCTOR

	#define SOFT_SHADOW_FUNC_BODY(shadowTerm, iLightType, bLinearSupport)
	float shadowTerm(float4 wPos)
	{
		if (__IsShadowMapValid_##shadowTerm##)
		{
			float4 lPos = mul(wPos,__ShadowMatrix_##shadowTerm##);
			return SoftEdgeShadowTerm(lPos,__InvShadowProjMat_##shadowTerm##,__ShadowMapSize_##shadowTerm##,__LightFrustum_##shadowTerm##,false,__DepthSamp_##shadowTerm##,__TransSamp_##shadowTerm##,__IndirCubeIndexSamp_##shadowTerm##,__FaceMatrixSamp_##shadowTerm##,iLightType,__vEVSMConfig_##shadowTerm##,bLinearSupport);
		}
		else return 1.0f;
	}

	#define SOFT_SHADOW_PARAM(shadowTerm, iLightType) 
	float4 __PlaceHolder_##iLightType##_##shadowTerm## : ShadowDS..;
	<
		string  UIName ="fake";
		string Object = "PointLight";
		string Space = "World";
		int refID = 0;
	>;

	float4x4 __ShadowMatrix_##shadowTerm##;
	float4x4 __InvShadowProjMat_##shadowTerm##;
	float4  __ShadowMapSize_##shadowTerm##;
	float4  __LightFrustum_##shadowTerm##;
	texture __DepthTex_##shadowTerm##;
	texture __TransTex_##shadowTerm##;
	texture __IndirCubeIndexTex_##shadowTerm##;
	texture __FaceMatrixTex_##shadowTerm##;
	bool    __IsTrans_##shadowTerm##;
	bool  __IsShadowMapValid_##shadowTerm##;
	float4  __vEVSMConfig_##shadowTerm##;

	sampler  __DepthSamp_##shadowTerm## = sampler_state
	{
		Texture = <__DepthTex_##shadowTerm##>;
		MinFilter= LINEAR;
		MagFilter = LINEAR; MipFilter = NONE;
		ADDRESSU = CLAMP;
		ADDRESSV = CLAMP; 
	};
	
	sampler  __TransSamp_##shadowTerm## = sampler_state
	{
		Texture= <__TransTex_##shadowTerm##>;
		MinFilter = LINEAR;
		MagFilter = LINEAR;
		MipFilter= NONE;
		ADDRESSU= CLAMP;
		ADDRESSV= CLAMP;
	};

	sampler __IndirCubeIndexSamp_##shadowTerm##= sampler_state
	{
		Texture =<__IndirCubeIndexTex_##shadowTerm##>;MinFilter= POINT; MagFilter = POINT; MipFilter = NONE;};
		
	sampler __FaceMatrixSamp_##shadowTerm##= sampler_state
	{Texture =<__FaceMatrixTex_##shadowTerm##>;MinFilter = POINT; MagFilter =POINT; MipFilter= NONE;};

	#define SOFT_SHADOW_FUNCTOR(functor,iLightType, bLinearSupport)\
	SOFT_SHADOW_PARAM(functor, iLightType);\
	SOFT_SHADOW_FUNC_BODY(functor, iLightType, bLinearSupport);
	
	#endif

even though I can’t as yet get soft shadows to work in an fx shader the following is quite useful

if you declare the following

SHADOW_FUNCTOR(shadowTerm);

and create the following function

float TshadowTerm(float4 wPos) 
{
	if (__IsShadowMapValid_shadowTerm)
	{
		float4 lPos = mul(wPos, __ShadowMatrix_shadowTerm);
		return ShadowTerm30(lPos,__ShadowMapSize_shadowTerm,__DepthSamp_shadowTerm, __TransSamp_shadowTerm,true, __IsTiledCube_shadowTerm,__IndirectCubeSamp_shadowTerm,__FaceDataSamp0_shadowTerm,__FaceDataSamp1_shadowTerm,__FaceDataMapOffset_shadowTerm);
	}
	else return 1.0f;
}

and use the following call in your pixel shader

float shadow =  TshadowTerm(Input.wPosition);

will enable Transparency in the shadows

good idea with the memory investigation and thanks a lot for looking into it.

It works if you activate Soft Shadows in the “Lighting and Shadows” dialog.

SOFT_SHADOW_FUNCTOR(shadowTerm, iLightType, bLinearSupport)

shadowTerm is the return value

iLightType is defined in the dxutil.dll
#define OMNI_LGT 000
#define SPOT_LGT 001
#define DIR_LGT 002

bLinearSupport is a bool (false or true)

a valid call would be:
SOFT_SHADOW_FUNCTOR(shadowTerm, DIR_LGT, true);

the value is fetched by calling:
shadowTerm(worldPosition)

There is still one left: AREA_SHADOW_FUNCTOR
Can’t get this one to work…