diff --git a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj index 80a470d..00e341b 100644 --- a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj +++ b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj @@ -15,6 +15,7 @@ SAK SAK SAK + DirectX11_2 1eec9909 @@ -28,7 +29,7 @@ 4 true false - true + false AnyCPU @@ -73,23 +74,23 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + $(SharpDXPackageBinDir)\SharpDX.D3DCompiler.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll @@ -109,6 +110,7 @@ ConsoleTextBox.cs + Form @@ -154,6 +156,14 @@ Content\DepthAndColorVS.cso PreserveNewest + + Content\frustumPS.cso + PreserveNewest + + + Content\frustumVS.cso + PreserveNewest + diff --git a/ProCamCalibration/CalibrateEnsemble/Frustum.cs b/ProCamCalibration/CalibrateEnsemble/Frustum.cs new file mode 100644 index 0000000..6b64e18 --- /dev/null +++ b/ProCamCalibration/CalibrateEnsemble/Frustum.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace RoomAliveToolkit +{ + using System; + using System.IO; + using System.Collections.Generic; + using System.Runtime.InteropServices; + using SharpDX; + using SharpDX.Direct3D11; + using SharpDX.DXGI; + using SharpDX.D3DCompiler; + using SharpDX.WIC; + using Device = SharpDX.Direct3D11.Device; + + public class FrustumShader + { + float maxZ = 0.97f; + + struct VertexPosition + { + public SharpDX.Vector4 position; + static public int SizeInBytes { get { return 4 * 4; } } + } + + + public SharpDX.Direct3D11.Buffer vertexBuffer; + public VertexBufferBinding vertexBufferBinding; + + public FrustumShader(Device device) + { + // create single vertex buffer + var stream = new DataStream(24 * VertexPosition.SizeInBytes, true, true); + stream.Write(new Vector4(-1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, -1, 0, 1)); + + stream.Write(new Vector4(-1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, -1, maxZ, 1)); + + stream.Write(new Vector4(-1, -1, 0, 1)); + stream.Write(new Vector4(-1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); + stream.Position = 0; + + var vertexBufferDesc = new BufferDescription() + { + BindFlags = BindFlags.VertexBuffer, + CpuAccessFlags = CpuAccessFlags.None, + Usage = ResourceUsage.Default, + SizeInBytes = 24 * VertexPosition.SizeInBytes, + }; + vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); + + stream.Dispose(); + + vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); + + + shaderByteCode = new ShaderBytecode(File.ReadAllBytes("Content/frustumVS.cso")); + frustumVS = new VertexShader(device, shaderByteCode); + frustumPS = new PixelShader(device, new ShaderBytecode(File.ReadAllBytes("Content/frustumPS.cso"))); + + // depth stencil state + var depthStencilStateDesc = new DepthStencilStateDescription() + { + IsDepthEnabled = true, + DepthWriteMask = DepthWriteMask.All, + DepthComparison = Comparison.LessEqual, + IsStencilEnabled = false, + }; + depthStencilState = new DepthStencilState(device, depthStencilStateDesc); + + // rasterizer state + var rasterizerStateDesc = new RasterizerStateDescription() + { + CullMode = CullMode.None, + FillMode = FillMode.Wireframe, + IsDepthClipEnabled = true, + IsFrontCounterClockwise = true, + IsMultisampleEnabled = true, + }; + rasterizerState = new RasterizerState(device, rasterizerStateDesc); + + // constant buffer + var VSConstantBufferDesc = new BufferDescription() + { + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ConstantBuffer, + SizeInBytes = VSConstantBuffer.size, + CpuAccessFlags = CpuAccessFlags.Write, + StructureByteStride = 0, + OptionFlags = 0, + }; + vertexShaderConstantBuffer = new SharpDX.Direct3D11.Buffer(device, VSConstantBufferDesc); + + // Pixel shader constant buffer + var PSConstantBufferDesc = new BufferDescription() + { + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ConstantBuffer, + SizeInBytes = PSConstantBuffer.size, + CpuAccessFlags = CpuAccessFlags.Write, + StructureByteStride = 0, + OptionFlags = 0, + }; + pixelShaderConstantBuffer = new SharpDX.Direct3D11.Buffer(device, PSConstantBufferDesc); + + vertexInputLayout = new InputLayout(device, shaderByteCode.Data, new[] + { + new InputElement("SV_POSITION", 0, Format.R32G32B32A32_Float, 0, 0), + }); + } + + // protip: compile shader with /Fc; output gives exact layout + // hlsl matrices are stored column major + // variables are stored on 4-component boundaries; inc. matrix columns + // size is a multiple of 16 + [StructLayout(LayoutKind.Explicit, Size = VSConstantBuffer.size)] + unsafe struct VSConstantBuffer + { + public const int size = 64 + 64 + 64; + + [FieldOffset(0)] + public fixed float world[16]; + [FieldOffset(64)] + public fixed float viewProjection[16]; + [FieldOffset(128)] + public fixed float inverseProjection[16]; + }; + + + [StructLayout(LayoutKind.Explicit, Size = PSConstantBuffer.size)] + unsafe struct PSConstantBuffer + { + public const int size = 16; + + [FieldOffset(0)] + public fixed float color[3]; + }; + + public unsafe void SetVertexShaderConstants(DeviceContext deviceContext, SharpDX.Matrix world, SharpDX.Matrix viewProjection, SharpDX.Matrix inverseProjection) + { + // hlsl matrices are default column order + var constants = new VSConstantBuffer(); + for (int i = 0, col = 0; col < 4; col++) + { + for (int row = 0; row < 4; row++) + { + constants.world[i] = world[row, col]; + constants.viewProjection[i] = viewProjection[row, col]; + constants.inverseProjection[i] = inverseProjection[row, col]; + i++; + } + } + + DataStream dataStream; + deviceContext.MapSubresource(vertexShaderConstantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.Write(constants); + deviceContext.UnmapSubresource(vertexShaderConstantBuffer, 0); + } + + public unsafe void SetPixelShaderConstants(DeviceContext deviceContext, SharpDX.Color3 color) + { + var constants = new PSConstantBuffer(); + for (int i = 0; i < 3; i++) + { + constants.color[i] = color[i]; + } + + DataStream dataStream; + deviceContext.MapSubresource(pixelShaderConstantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.Write(constants); + deviceContext.UnmapSubresource(pixelShaderConstantBuffer, 0); + } + + public void Render(DeviceContext deviceContext) + { + deviceContext.InputAssembler.InputLayout = vertexInputLayout; + deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.LineList; + deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); + deviceContext.Rasterizer.State = rasterizerState; + + deviceContext.VertexShader.Set(frustumVS); + deviceContext.VertexShader.SetConstantBuffer(0, vertexShaderConstantBuffer); + + deviceContext.GeometryShader.Set(null); + + deviceContext.PixelShader.SetConstantBuffer(0, pixelShaderConstantBuffer); + deviceContext.PixelShader.Set(frustumPS); + + deviceContext.Draw(24, 0); + } + + VertexShader frustumVS; + PixelShader frustumPS; + ShaderBytecode shaderByteCode; + DepthStencilState depthStencilState; + RasterizerState rasterizerState; + SharpDX.Direct3D11.Buffer vertexShaderConstantBuffer; + SharpDX.Direct3D11.Buffer pixelShaderConstantBuffer; + InputLayout vertexInputLayout; + } +} diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs index 156c1a7..8d87ec2 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs @@ -52,6 +52,10 @@ private void InitializeComponent() this.renderToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.liveViewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.toolStripSeparator2 = new System.Windows.Forms.ToolStripSeparator(); + this.wireframeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.cameraColorFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.cameraDepthFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.projectorFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.perspectiveAtOriginToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.toolStripSeparator3 = new System.Windows.Forms.ToolStripSeparator(); @@ -239,6 +243,10 @@ private void InitializeComponent() // this.renderToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { this.liveViewToolStripMenuItem, + this.wireframeToolStripMenuItem, + this.cameraColorFrustumsToolStripMenuItem, + this.cameraDepthFrustumsToolStripMenuItem, + this.projectorFrustumsToolStripMenuItem, this.toolStripSeparator2}); this.renderToolStripMenuItem.Name = "renderToolStripMenuItem"; this.renderToolStripMenuItem.Size = new System.Drawing.Size(56, 20); @@ -247,14 +255,42 @@ private void InitializeComponent() // liveViewToolStripMenuItem // this.liveViewToolStripMenuItem.Name = "liveViewToolStripMenuItem"; - this.liveViewToolStripMenuItem.Size = new System.Drawing.Size(123, 22); + this.liveViewToolStripMenuItem.Size = new System.Drawing.Size(180, 22); this.liveViewToolStripMenuItem.Text = "Live View"; this.liveViewToolStripMenuItem.Click += new System.EventHandler(this.liveViewToolStripMenuItem_Click); // // toolStripSeparator2 // this.toolStripSeparator2.Name = "toolStripSeparator2"; - this.toolStripSeparator2.Size = new System.Drawing.Size(120, 6); + this.toolStripSeparator2.Size = new System.Drawing.Size(177, 6); + // + // wireframeToolStripMenuItem + // + this.wireframeToolStripMenuItem.Name = "wireframeToolStripMenuItem"; + this.wireframeToolStripMenuItem.Size = new System.Drawing.Size(74, 20); + this.wireframeToolStripMenuItem.Text = "Wireframe"; + this.wireframeToolStripMenuItem.Click += new System.EventHandler(this.wireframeViewToolStripMenuItem_Click); + // + // cameraColorFrustumsToolStripMenuItem + // + this.cameraColorFrustumsToolStripMenuItem.Name = "cameraColorFrustumsToolStripMenuItem"; + this.cameraColorFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); + this.cameraColorFrustumsToolStripMenuItem.Text = "Camera Color Frustums"; + this.cameraColorFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraColorFrustumToolStripMenuItem_Click); + // + // cameraDepthFrustumsToolStripMenuItem + // + this.cameraDepthFrustumsToolStripMenuItem.Name = "cameraDepthFrustumsToolStripMenuItem"; + this.cameraDepthFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); + this.cameraDepthFrustumsToolStripMenuItem.Text = "Camera Depth Frustums"; + this.cameraDepthFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraDepthFrustumToolStripMenuItem_Click); + // + // projectorFrustumsToolStripMenuItem + // + this.projectorFrustumsToolStripMenuItem.Name = "projectorFrustumsToolStripMenuItem"; + this.projectorFrustumsToolStripMenuItem.Size = new System.Drawing.Size(119, 20); + this.projectorFrustumsToolStripMenuItem.Text = "Projector Frustums"; + this.projectorFrustumsToolStripMenuItem.Click += new System.EventHandler(this.projectorFrustumToolStripMenuItem_Click); // // viewToolStripMenuItem // @@ -402,5 +438,9 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripMenuItem decodeGrayCodesToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem calibrateProjectorGroupsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem optimizePoseToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem wireframeToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem cameraColorFrustumsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem cameraDepthFrustumsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem projectorFrustumsToolStripMenuItem; } } \ No newline at end of file diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 9feddfe..96061d4 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; +using System.ServiceModel; using System.ServiceModel.Discovery; using System.Windows.Forms; @@ -28,6 +29,7 @@ public MainForm(string[] args) this.args = args; } + FrustumShader frustumShader; ProjectorCameraEnsemble ensemble; string path, directory; string[] args; @@ -77,6 +79,9 @@ protected override void OnLoad(EventArgs e) // viewport viewport = new Viewport(0, 0, videoPanel1.Width, videoPanel1.Height, 0f, 1f); + + frustumShader = new FrustumShader(device); + // shaders var shaderByteCode = new ShaderBytecode(File.ReadAllBytes("Content/DepthAndColorVS.cso")); depthAndColorVS2 = new VertexShader(device, shaderByteCode); @@ -93,16 +98,7 @@ protected override void OnLoad(EventArgs e) }; depthStencilState = new DepthStencilState(device, depthStencilStateDesc); - // rasterizer states - var rasterizerStateDesc = new RasterizerStateDescription() - { - CullMode = CullMode.None, // beware what this does to both shaders - FillMode = FillMode.Solid, - IsDepthClipEnabled = true, - IsFrontCounterClockwise = true, - IsMultisampleEnabled = true, - }; - rasterizerState = new RasterizerState(device, rasterizerStateDesc); + UpdateRasterizerState(); // color sampler state var colorSamplerStateDesc = new SamplerStateDescription() @@ -160,6 +156,23 @@ protected override void OnLoad(EventArgs e) new System.Threading.Thread(RenderLoop).Start(); } + private void UpdateRasterizerState() + { + lock (renderLock) + { + // rasterizer states + var rasterizerStateDesc = new RasterizerStateDescription() + { + CullMode = CullMode.None, // beware what this does to both shaders + FillMode = wireframe ? FillMode.Wireframe : FillMode.Solid, + IsDepthClipEnabled = true, + IsFrontCounterClockwise = true, + IsMultisampleEnabled = true, + }; + rasterizerState = new RasterizerState(device, rasterizerStateDesc); + } + } + SharpDX.Direct3D11.Device device; SwapChain swapChain; Texture2D renderTarget, depthStencil; @@ -422,31 +435,39 @@ void ColorCameraLoop() { while (live) { - var encodedColorData = camera.Client.LatestJPEGImage(); - - // decode JPEG - var memoryStream = new MemoryStream(encodedColorData); - var stream = new WICStream(imagingFactory, memoryStream); - // decodes to 24 bit BGR - var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); - var bitmapFrameDecode = decoder.GetFrame(0); - - // convert to 32 bpp - var formatConverter = new FormatConverter(imagingFactory); - formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); - formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory - //lock (colorData) - // Swap(ref colorData, ref nextColorData); - lock (renderLock) // necessary? + try { - UpdateColorImage(device.ImmediateContext, nextColorData); + var encodedColorData = camera.Client.LatestJPEGImage(); + + // decode JPEG + var memoryStream = new MemoryStream(encodedColorData); + var stream = new WICStream(imagingFactory, memoryStream); + // decodes to 24 bit BGR + var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); + var bitmapFrameDecode = decoder.GetFrame(0); + + // convert to 32 bpp + var formatConverter = new FormatConverter(imagingFactory); + formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); + formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory + //lock (colorData) + // Swap(ref colorData, ref nextColorData); + lock (renderLock) // necessary? + { + UpdateColorImage(device.ImmediateContext, nextColorData); + } + memoryStream.Close(); + memoryStream.Dispose(); + stream.Dispose(); + decoder.Dispose(); + formatConverter.Dispose(); + bitmapFrameDecode.Dispose(); + } + catch (EndpointNotFoundException) + { + Console.WriteLine("Could not find Kinect server for live color update."); + live = false; } - memoryStream.Close(); - memoryStream.Dispose(); - stream.Dispose(); - decoder.Dispose(); - formatConverter.Dispose(); - bitmapFrameDecode.Dispose(); } } @@ -456,12 +477,21 @@ void DepthCameraLoop() { while (live) { - nextDepthData = camera.Client.LatestDepthImage(); - //lock (remoteDepthData) - // Swap(ref remoteDepthData, ref nextRemoteDepthData); - lock (renderLock) + try { - UpdateDepthImage(device.ImmediateContext, nextDepthData); + nextDepthData = camera.Client.LatestDepthImage(); + + //lock (remoteDepthData) + // Swap(ref remoteDepthData, ref nextRemoteDepthData); + lock (renderLock) + { + UpdateDepthImage(device.ImmediateContext, nextDepthData); + } + } + catch (EndpointNotFoundException) + { + Console.WriteLine("Could not find Kinect server for live depth update."); + live = false; } } } @@ -502,12 +532,16 @@ public unsafe void SetConstants(DeviceContext deviceContext, RoomAliveToolkit.Ki // hlsl matrices are default column order var constants = new ConstantBuffer(); for (int i = 0, col = 0; col < 4; col++) + { for (int row = 0; row < 4; row++) { constants.projection[i] = projection[row, col]; constants.depthToColorTransform[i] = (float)kinect2Calibration.depthToColorTransform[row, col]; i++; } + } + + constants.f[0] = (float)kinect2Calibration.colorCameraMatrix[0, 0]; constants.f[1] = (float)kinect2Calibration.colorCameraMatrix[1, 1]; constants.c[0] = (float)kinect2Calibration.colorCameraMatrix[0, 2]; @@ -533,7 +567,6 @@ void RenderLoop() view = manipulator.Update(); var deviceContext = device.ImmediateContext; - deviceContext.InputAssembler.InputLayout = vertexInputLayout; deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.TriangleList; deviceContext.OutputMerger.SetTargets(depthStencilView, renderTargetView); @@ -550,9 +583,13 @@ void RenderLoop() // render all cameras if (ensemble != null) + { + var viewProjection = view * projection; + foreach (var camera in ensemble.cameras) { if (cameraDeviceResources.ContainsKey(camera)) + { if (cameraDeviceResources[camera].renderEnabled && (camera.pose != null)) { var world = new SharpDX.Matrix(); @@ -562,13 +599,80 @@ void RenderLoop() world.Transpose(); // view and projection matrix are post-multiply - var worldViewProjection = world * view * projection; + var worldViewProjection = world * viewProjection; SetConstants(deviceContext, camera.calibration, worldViewProjection); cameraDeviceResources[camera].Render(deviceContext); } + } + } + + foreach (var camera in ensemble.cameras) + { + if (cameraDeviceResources.ContainsKey(camera)) + { + if (cameraDeviceResources[camera].renderEnabled && (camera.pose != null)) + { + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)camera.pose[i, j]; + world.Transpose(); + + // view and projection matrix are post-multiply + var worldViewProjection = world * viewProjection; + + if (renderCameraColorFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 1f, 0f)); + + SharpDX.Matrix cameraProjection = GetCameraColorProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } + + if (renderCameraDepthFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(0f, 1f, 1f)); + + SharpDX.Matrix cameraProjection = GetCameraDepthProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } + } + } } + if (renderProjectorFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 0f, 1f)); + foreach (var projector in ensemble.projectors) + { + if (projector.pose != null) + { + SharpDX.Matrix projectorProjection = GetProjectorProjectionMatrix(projector); + projectorProjection.Transpose(); + projectorProjection.Invert(); + + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)projector.pose[i, j]; + world.Transpose(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, projectorProjection); + frustumShader.Render(deviceContext); + } + } + } + } + swapChain.Present(1, PresentFlags.None); } } @@ -648,7 +752,13 @@ void renderMenuItem_Click(object sender, EventArgs e) var toolStripMenuItem = (ToolStripMenuItem)sender; var camera = (ProjectorCameraEnsemble.Camera)toolStripMenuItem.Tag; toolStripMenuItem.Checked = !toolStripMenuItem.Checked; - cameraDeviceResources[camera].renderEnabled = toolStripMenuItem.Checked; + + // Prevents KeyNotFoundException if camera was not calibrated and therefore CameraDeviceResource was not created + CameraDeviceResource cameraDeviceResource; + if (cameraDeviceResources.TryGetValue(camera, out cameraDeviceResource)) + { + cameraDeviceResource.renderEnabled = toolStripMenuItem.Checked; + } } bool perspectiveView; @@ -671,7 +781,8 @@ void viewMenuItem_Click(object sender, EventArgs e) toolStripMenuItem.Checked = true; var projector = (ProjectorCameraEnsemble.Projector)toolStripMenuItem.Tag; - SetViewProjectionFromProjector(projector); + projection = GetProjectorProjectionMatrix(projector); + projection.Transpose(); manipulator.View = view; manipulator.OriginalView = view; perspectiveView = false; @@ -909,6 +1020,35 @@ private void liveViewToolStripMenuItem_Click(object sender, EventArgs e) liveViewToolStripMenuItem.Checked = live; } + bool wireframe = false; + private void wireframeViewToolStripMenuItem_Click(object sender, EventArgs e) + { + wireframe = !wireframe; + wireframeToolStripMenuItem.Checked = wireframe; + UpdateRasterizerState(); + } + + bool renderCameraColorFrustums = false; + private void cameraColorFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderCameraColorFrustums = !renderCameraColorFrustums; + cameraColorFrustumsToolStripMenuItem.Checked = renderCameraColorFrustums; + } + + bool renderCameraDepthFrustums = false; + private void cameraDepthFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderCameraDepthFrustums = !renderCameraDepthFrustums; + cameraDepthFrustumsToolStripMenuItem.Checked = renderCameraDepthFrustums; + } + + bool renderProjectorFrustums = false; + private void projectorFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderProjectorFrustums = !renderProjectorFrustums; + projectorFrustumsToolStripMenuItem.Checked = renderProjectorFrustums; + } + private void videoPanel1_SizeChanged(object sender, EventArgs e) { // TODO: look into using this as initial creation @@ -1094,6 +1234,7 @@ void LoadEnsemble() { ensemble = ProjectorCameraEnsemble.FromFile(path); Console.WriteLine("Loaded " + path); + Console.WriteLine("Containing: {0} camera(s), {1} projector(s)", ensemble.cameras.Count, ensemble.projectors.Count); } catch (Exception ex) { @@ -1194,7 +1335,7 @@ void AcquireDepthAndColor() // could be method on Projector: - void SetViewProjectionFromProjector(ProjectorCameraEnsemble.Projector projector) + SharpDX.Matrix GetProjectorProjectionMatrix(ProjectorCameraEnsemble.Projector projector) { if ((projector.pose == null) || (projector.cameraMatrix == null)) Console.WriteLine("Projector pose/camera matrix not set. Please perform a calibration."); @@ -1220,14 +1361,15 @@ void SetViewProjectionFromProjector(ProjectorCameraEnsemble.Projector projector) float w = projector.width; float h = projector.height; - projection = GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); - projection.Transpose(); + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); } + + return SharpDX.Matrix.Identity; } - void SetViewProjectionFromCamera(ProjectorCameraEnsemble.Camera camera) + SharpDX.Matrix GetCameraColorProjectionMatrix(ProjectorCameraEnsemble.Camera camera) { - if ((camera.pose == null) || (camera.calibration.colorCameraMatrix == null)) + if ((camera.pose == null) || (camera.calibration == null)) Console.WriteLine("Camera pose not set."); else { @@ -1250,9 +1392,41 @@ void SetViewProjectionFromCamera(ProjectorCameraEnsemble.Camera camera) float w = Kinect2Calibration.colorImageWidth; float h = Kinect2Calibration.colorImageHeight; - projection = GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); - projection.Transpose(); + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); + } + + return SharpDX.Matrix.Identity; + } + + SharpDX.Matrix GetCameraDepthProjectionMatrix(ProjectorCameraEnsemble.Camera camera) + { + if ((camera.pose == null) || (camera.calibration == null)) + Console.WriteLine("Camera pose not set."); + else + { + view = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + view[i, j] = (float)camera.pose[i, j]; + view.Invert(); + view.Transpose(); + + var cameraMatrix = camera.calibration.depthCameraMatrix; + float fx = (float)cameraMatrix[0, 0]; + float fy = (float)cameraMatrix[1, 1]; + float cx = (float)cameraMatrix[0, 2]; + float cy = (float)cameraMatrix[1, 2]; + + float near = 0.1f; + float far = 100.0f; + + float w = Kinect2Calibration.depthImageWidth; + float h = Kinect2Calibration.depthImageHeight; + + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); } + + return SharpDX.Matrix.Identity; } void SetDefaultView() diff --git a/ProCamCalibration/KinectServer/KinectServer.csproj b/ProCamCalibration/KinectServer/KinectServer.csproj index 44d02d5..17b2968 100644 --- a/ProCamCalibration/KinectServer/KinectServer.csproj +++ b/ProCamCalibration/KinectServer/KinectServer.csproj @@ -16,6 +16,7 @@ SAK SAK + DirectX11_2 0eef0576 @@ -27,7 +28,7 @@ DEBUG;TRACE prompt 4 - true + false false @@ -69,11 +70,11 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll diff --git a/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj b/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj index 10b0743..96c67b0 100644 --- a/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj +++ b/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj @@ -15,7 +15,9 @@ SAK SAK SAK - e31ed761 + DirectX11_2 + + true @@ -26,7 +28,7 @@ prompt 4 true - true + false pdbonly @@ -45,17 +47,21 @@ ..\packages\Microsoft.Kinect.2.0.1410.19000\lib\net45\Microsoft.Kinect.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + + False + $(SharpDXPackageBinDir)\SharpDX.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + + False + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + + False + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + + False + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/ProjectionMappingSample/MeshShader.cs b/ProCamCalibration/ProjectionMappingSample/MeshShader.cs index e0f51f0..b1fe305 100644 --- a/ProCamCalibration/ProjectionMappingSample/MeshShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/MeshShader.cs @@ -289,7 +289,6 @@ public void Render(DeviceContext deviceContext, MeshDeviceResources meshDeviceRe deviceContext.VertexShader.Set(meshVS); deviceContext.VertexShader.SetConstantBuffer(0, vertexShaderConstantBuffer); deviceContext.GeometryShader.Set(null); - deviceContext.PixelShader.Set(meshPS); deviceContext.PixelShader.SetSampler(0, colorSamplerState); deviceContext.PixelShader.SetConstantBuffer(0, pixelShaderConstantBuffer); deviceContext.OutputMerger.SetTargets(depthStencilView, renderTargetView); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj index b7c5f1b..edbdea1 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj @@ -5,17 +5,19 @@ Debug AnyCPU {59A35887-D3F7-4AB8-862F-98F68F9F7B51} - Exe + WinExe Properties RoomAliveToolkit ProjectionMappingSample - v4.5 + v4.5.1 512 SAK SAK SAK SAK + DirectX11_2 d0c2a9ee + AnyCPU @@ -27,7 +29,7 @@ prompt 4 true - true + false false @@ -52,19 +54,19 @@ ..\packages\Microsoft.Kinect.2.0.1410.19000\lib\net45\Microsoft.Kinect.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + $(SharpDXPackageBinDir)\SharpDX.D3DCompiler.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/ProjectorServer/ProjectorServer.cs b/ProCamCalibration/ProjectorServer/ProjectorServer.cs index eebb930..183fbbd 100644 --- a/ProCamCalibration/ProjectorServer/ProjectorServer.cs +++ b/ProCamCalibration/ProjectorServer/ProjectorServer.cs @@ -33,6 +33,7 @@ public void OpenDisplay(int screenIndex) { var projectorForm = new ProjectorForm(screenIndex); projectorForm.Show(); + projectorForm.BringToFront(); projectorForms[screenIndex] = projectorForm; } } @@ -75,6 +76,7 @@ public int NumberOfGrayCodeImages(int screenIndex) public void DisplayGrayCode(int screenIndex, int i) { var projectorForm = projectorForms[screenIndex]; + projectorForm.BringToFront(); projectorForm.DisplayGrayCode(i); } diff --git a/ProCamCalibration/ProjectorServer/ProjectorServer.csproj b/ProCamCalibration/ProjectorServer/ProjectorServer.csproj index 60d9c4f..51c7773 100644 --- a/ProCamCalibration/ProjectorServer/ProjectorServer.csproj +++ b/ProCamCalibration/ProjectorServer/ProjectorServer.csproj @@ -16,6 +16,7 @@ SAK SAK + DirectX11_2 0eef0576 @@ -27,7 +28,7 @@ DEBUG;TRACE prompt 4 - true + false false @@ -69,15 +70,15 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/Shaders/Shaders.vcxproj b/ProCamCalibration/Shaders/Shaders.vcxproj index d268a35..ab3f2bb 100644 --- a/ProCamCalibration/Shaders/Shaders.vcxproj +++ b/ProCamCalibration/Shaders/Shaders.vcxproj @@ -37,7 +37,7 @@ Application true - v120 + v140 Unicode @@ -210,6 +210,18 @@ 5.0 Pixel + + Pixel + Pixel + Pixel + Pixel + + + Vertex + Vertex + Vertex + Vertex + Vertex 5.0 diff --git a/ProCamCalibration/Shaders/Shaders.vcxproj.filters b/ProCamCalibration/Shaders/Shaders.vcxproj.filters index 58be562..ea4aaa7 100644 --- a/ProCamCalibration/Shaders/Shaders.vcxproj.filters +++ b/ProCamCalibration/Shaders/Shaders.vcxproj.filters @@ -46,5 +46,11 @@ Source Files + + Source Files + + + Source Files + \ No newline at end of file diff --git a/ProCamCalibration/Shaders/frustumPS.hlsl b/ProCamCalibration/Shaders/frustumPS.hlsl new file mode 100644 index 0000000..bd960a5 --- /dev/null +++ b/ProCamCalibration/Shaders/frustumPS.hlsl @@ -0,0 +1,14 @@ +struct PSInput +{ + float4 pos : SV_POSITION; +}; + +cbuffer constants : register(b0) +{ + float3 color; +} + +float4 main(PSInput input) : SV_TARGET0 +{ + return float4(color, 1.0); +} diff --git a/ProCamCalibration/Shaders/frustumVS.hlsl b/ProCamCalibration/Shaders/frustumVS.hlsl new file mode 100644 index 0000000..6f78e4f --- /dev/null +++ b/ProCamCalibration/Shaders/frustumVS.hlsl @@ -0,0 +1,26 @@ +struct VSInput +{ + float4 pos : SV_POSITION; +}; + +cbuffer constants : register(b0) +{ + matrix world; + matrix viewProjection; + matrix inverseProjection; +} + +struct VSOutput +{ + float4 pos : SV_POSITION; +}; + +VSOutput main(VSInput input) +{ + VSOutput output = (VSOutput)0; + + float4 invPos = mul(input.pos, inverseProjection); + float4 worldPos = mul(invPos, world); + output.pos = mul(worldPos, viewProjection); + return output; +} \ No newline at end of file