HLSL/SAS, 3dsmax and environment effects

Hi everyone !

I’m playing with HLSL shaders and SAS scripting, trying to use these as much as possible in 3dsmax (it is an enjoyable task if you omit the lack of documentation, the somehow “original” interpretation of SAS in 3dsmax, the bugs maybe and so on…). My subject of interest nowadays is effects which need whole scene render in render targets, possibly from various view points. Using the Scene Effect Loader and patience (a lot), I succeeded to render DOF, shadow mapping for one light, etc… Yeepee.

But I came into troubles when I tried to render more than one effect at once (DOF and shadow mapping for example).
So I re-started from the beginning and tried to achieve the following:

  1. render a simple scene (say a teapot) in two RT cleared in blue, using two different vertex shaders (just to simulate real conditions), and two very simple pixel shaders (one which renders the objet in red and the other in green),
  2. then use a full screen quad render using these two RT and a compositing pixel shader (which sums up the two RT… art masterpiece).

After a few tries, the best I managed to get is this:

when I expected that:

Here are the shaders:

EnvTEST.fx


string ParamID = "0x003";

float4x4 WorldViewProj : WORLDVIEWPROJ;

struct SInputVS
{
        float3 Pos : POSITION;
};

struct SOutputVS
{
        float4 Pos : POSITION;
};

SOutputVS TEST_VS( SInputVS input )
{
        SOutputVS output;
        output.Pos = mul(float4(input.Pos, 1.0f), WorldViewProj);
        return output;
}

SOutputVS TEST_Scaled_VS( SInputVS input )
{
        SOutputVS output;
        output.Pos = mul(float4(input.Pos*1.5f, 1.0f), WorldViewProj);
        return output;
}

float4 TEST_Red_PS( SOutputVS input ) : COLOR
{
        return float4(1.0f,0.0f,0.0f,1.0f);
}

float4 TEST_Green_PS( SOutputVS input ) : COLOR
{
        return float4(0.0f,1.0f,0.0f,1.0f);
}


texture RedRender : RENDERCOLORTARGET
<
        float2 ViewPortRatio = { 1.0f, 1.0f };
        int MipLevels = 1;
        string Format = "A8R8G8B8";
        string UIWidget = "None";
>;

sampler RedRenderSampler = sampler_state
{
        texture = <RedRender>;
        AddressU  = CLAMP;
        AddressV = CLAMP;
        MipFilter = POINT;
        MinFilter = LINEAR;
        MagFilter = LINEAR;
};

texture GreenRender : RENDERCOLORTARGET
<
        float2 ViewPortRatio = { 1.0f, 1.0f };
        int MipLevels = 1;
        string Format = "A8R8G8B8";
        string UIWidget = "None";
>;

sampler GreenRenderSampler = sampler_state
{
        texture = <GreenRender>;
        AddressU  = CLAMP;
        AddressV = CLAMP;
        MipFilter = POINT;
        MinFilter = LINEAR;
        MagFilter = LINEAR;
};

float4 ClearColor = float4(0.0f, 0.0f, 1.0f, 1.0f);


float Script : STANDARDSGLOBAL
<
        string UIWidget = "none";
        string ScriptClass = "scene";
        string ScriptOrder = "preprocess";
        string ScriptOutput = "color";
        string Script = "Technique=TEST;";
> = 0.8;

technique TEST
<
        string ScriptClass = "scene";
        string ScriptOrder = "preprocess";
        string ScriptOutput = "color";
        string Script = 
                "RenderColorTarget0=RedRender;"
                "RenderDepthStencilTarget=;"
                "ClearSetColor=ClearColor;"
                "ClearSetDepth=1.0f;"
                "Clear=Color;"
                "Clear=Depth;"
                "Pass=Red;"

                "RenderColorTarget0=GreenRender;"
                "RenderDepthStencilTarget=;"
                "ClearSetColor=ClearColor;"
                "ClearSetDepth=1.0f;"
                "Clear=Color;"
                "Clear=Depth;"
                "Pass=Green;";
>
{
        pass Red
        <
                string Script =
                        "Draw=Geometry;";
        >
        {
                ZEnable = false;
                CullMode = CW;
                AlphaBlendEnable = false;
                VertexShader = compile vs_3_0 TEST_Scaled_VS();
                PixelShader = compile ps_3_0 TEST_Red_PS();
        }

        pass Green
        <
                string Script =
                        "Draw=Geometry;";
        >
        {
                ZEnable = false;
                CullMode = CW;
                AlphaBlendEnable = false;
                VertexShader = compile vs_3_0 TEST_VS();
                PixelShader = compile ps_3_0 TEST_Green_PS();
        }
}

PostTEST.fx

string ParamID = "0x003";

float4x4 WorldViewProj : WORLDVIEWPROJ;

struct SInputVS
{
        float3 Pos : POSITION;
};

struct SOutputVS
{
        float4 Pos : POSITION;
};

SOutputVS TEST_VS( SInputVS input )
{
        SOutputVS output;
        output.Pos = mul(float4(input.Pos, 1.0f), WorldViewProj);
        return output;
}

SOutputVS TEST_Scaled_VS( SInputVS input )
{
        SOutputVS output;
        output.Pos = mul(float4(input.Pos*1.5f, 1.0f), WorldViewProj);
        return output;
}

float4 TEST_Red_PS( SOutputVS input ) : COLOR
{
        return float4(1.0f,0.0f,0.0f,1.0f);
}

float4 TEST_Green_PS( SOutputVS input ) : COLOR
{
        return float4(0.0f,1.0f,0.0f,1.0f);
}


texture RedRender : RENDERCOLORTARGET
<
        float2 ViewPortRatio = { 1.0f, 1.0f };
        int MipLevels = 1;
        string Format = "A8R8G8B8";
        string UIWidget = "None";
>;

sampler RedRenderSampler = sampler_state
{
        texture = <RedRender>;
        AddressU  = CLAMP;
        AddressV = CLAMP;
        MipFilter = POINT;
        MinFilter = LINEAR;
        MagFilter = LINEAR;
};

texture GreenRender : RENDERCOLORTARGET
<
        float2 ViewPortRatio = { 1.0f, 1.0f };
        int MipLevels = 1;
        string Format = "A8R8G8B8";
        string UIWidget = "None";
>;

sampler GreenRenderSampler = sampler_state
{
        texture = <GreenRender>;
        AddressU  = CLAMP;
        AddressV = CLAMP;
        MipFilter = POINT;
        MinFilter = LINEAR;
        MagFilter = LINEAR;
};

float4 ClearColor = float4(0.0f, 0.0f, 1.0f, 1.0f);


float Script : STANDARDSGLOBAL
<
        string UIWidget = "none";
        string ScriptClass = "scene";
        string ScriptOrder = "preprocess";
        string ScriptOutput = "color";
        string Script = "Technique=TEST;";
> = 0.8;

technique TEST
<
        string ScriptClass = "scene";
        string ScriptOrder = "preprocess";
        string ScriptOutput = "color";
        string Script = 
                "RenderColorTarget0=RedRender;"
                "RenderDepthStencilTarget=;"
                "ClearSetColor=ClearColor;"
                "ClearSetDepth=1.0f;"
                "Clear=Color;"
                "Clear=Depth;"
                "Pass=Red;"

                "RenderColorTarget0=GreenRender;"
                "RenderDepthStencilTarget=;"
                "ClearSetColor=ClearColor;"
                "ClearSetDepth=1.0f;"
                "Clear=Color;"
                "Clear=Depth;"
                "Pass=Green;";
>
{
        pass Red
        <
                string Script =
                        "Draw=Geometry;";
        >
        {
                ZEnable = false;
                CullMode = CW;
                AlphaBlendEnable = false;
                VertexShader = compile vs_3_0 TEST_Scaled_VS();
                PixelShader = compile ps_3_0 TEST_Red_PS();
        }

        pass Green
        <
                string Script =
                        "Draw=Geometry;";
        >
        {
                ZEnable = false;
                CullMode = CW;
                AlphaBlendEnable = false;
                VertexShader = compile vs_3_0 TEST_VS();
                PixelShader = compile ps_3_0 TEST_Green_PS();
        }
}

As you can see the shaders are pretty simple and the SAS scripting is not complex… What I’ve found is that both targets are properly accessible and cleared to the blue color. But the red rendering pass, if actually done, is done in the wrong render target, even though the script commands are properly ordered (in my opinion at least :)). So I get a fully blue image and another with blue background and both teapots rendered: the big red with the small green inside… Sic.

Oh and I almost forgot: for that to work, the object has to be assigned a DirectX shader (say StandardFX.fx) for the above to work… Is actually don’t understand why, maybe it’s a clue ?

At a moment I thought I found the solution: an esoteric parameters of Scene Effect Loader, accessible using max scripting, numberOfPasses was supposed to be used to define the number of passes needed in environment effect (the doc says “6 if you render a cube map”). Very well, I wrote the script below (you’ve already understood it doesn’t work, but I add it here in order to be as complete as possible):


struct loadTEST
(
        fn BuildFullSceneFxPath _inFxName =
        (
                path = "C:\Projects\depot\TEST\Media\FXs" + _inFxName
        ),

        fn Init = 
        (
                -- Pre-process & environment shaders
                pathPre = BuildFullSceneFxPath "\EnvTEST.fx"
                SceneEffectLoader.LoadSceneEffect pathPre effectType:#Env numberOfPasses:2
        )
)

loadTESTInstance = loadTEST()
loadTESTInstance.Init()

That’s the point I’ve reached so far. So if anyone can give me an hint I would be really gratefull… Any solution is accepted, including a “it’s not possible dude” :). But as I’ve already been near to think that for a certain amount of old problems using SAS & 3dsmax, which I managed to solve…

Anyway, thanks to everyone who’s been patient enough to read that until the end (thanks also to all the other who gave up earlier… ;)).

I don’t have the answer. I stopped messing with sceneFX in earlier versions of max because I couldn’t get them to work very well.
I believe in later versions things have been improved but I haven’t look at it much.

But good job on getting this far!
With the lack of documentation on them, I’m impressed you got things like DOF to work!

Thanks Kees, if someone like you doesn’t know how to do it, then I think I can stop my investigations right now… Btw, I take this opportunity to thank you for all your posts around the web about HLSL and SAS, you (and others like Ben Cloward and co) have often lit the way in 3dsmax’s HLSL darkness :wink:

Thanks!
I haven’t looked into post FX shaders in 3dsmax that much really. So I wouldn’t take me not knowing the answer as a reason to stop looking into it! :slight_smile:

I do know that postFX shaders in max USED to be not as complete and functioning as regular pixel shaders. I don’t know if that has improved in recent builds.

The thing with these post-FX shaders is that they rely a lot on how things are supported in the 3d-app core. Like you already found out. The order in which you do things or the way you try to combine them, can really work differently then you would think.

You’re right about ordering or combinaison (even though as we’re talking about a standardized scripting language, we should not have these kind of problems. Ah ! the magic of SAS in 3dsmax…). I have the feeling that I’ve pretty much tried all the possible permutations, declarations using techniques or passes, etc… to try and get the expected results.
At least all that came to my mind, I might have forgotten one or the other solution but I think I’ll use a different approach.
Nethertheless, thanks a lot ! And althought I’m pretty sure you’ll find them obvious if someone wants a few tips about DOF or effects like this one, simply send me a message (until you don’t want to mix several effects at once :wink:)
Cheers.