From d79f0dc73b3392a8cd902ef0611ce77f90a67967 Mon Sep 17 00:00:00 2001 From: loki Date: Thu, 9 Jan 2020 12:12:18 +0100 Subject: [PATCH] Capture Desktop images on Windows --- sunshine/platform/common.h | 18 +- sunshine/platform/linux.cpp | 69 +++--- sunshine/platform/linux_evdev.cpp | 4 +- sunshine/platform/windows.cpp | 388 +++++++++++++++++++++++------- sunshine/stream.cpp | 3 + sunshine/video.cpp | 34 ++- 6 files changed, 377 insertions(+), 139 deletions(-) diff --git a/sunshine/platform/common.h b/sunshine/platform/common.h index 770d2a9b..0969972a 100644 --- a/sunshine/platform/common.h +++ b/sunshine/platform/common.h @@ -16,12 +16,22 @@ public: std::int32_t width; std::int32_t height; - virtual ~img_t() {}; + virtual ~img_t() = default; +}; + +enum class capture_e : int { + ok, + reinit, + timeout, + error }; class display_t { public: - virtual std::unique_ptr snapshot(bool cursor) = 0; + virtual capture_e snapshot(std::unique_ptr &img, bool cursor) = 0; + virtual int reinit() = 0; + virtual std::unique_ptr alloc_img() = 0; + virtual ~display_t() = default; }; @@ -34,12 +44,12 @@ public: void freeInput(void*); -using input_t = util::safe_ptr; +using input_t = util::safe_ptr; std::string get_local_ip(); std::unique_ptr microphone(); -std::unique_ptr display(); +std::shared_ptr display(); input_t input(); void move_mouse(input_t &input, int deltaX, int deltaY); diff --git a/sunshine/platform/linux.cpp b/sunshine/platform/linux.cpp index 831dbb9a..b971cf15 100644 --- a/sunshine/platform/linux.cpp +++ b/sunshine/platform/linux.cpp @@ -75,12 +75,6 @@ public: }; struct x11_img_t : public img_t { - x11_img_t(std::uint8_t *data, std::int32_t width, std::int32_t height, XImage *img) : img { img } { - this->data = data; - this->width = width; - this->height = height; - } - ximg_t img; }; @@ -141,7 +135,7 @@ struct x11_attr_t : public display_t { XGetWindowAttributes(xdisplay.get(), xwindow, &xattr); } - std::unique_ptr snapshot(bool cursor) override { + capture_e snapshot(std::unique_ptr &img_out_base, bool cursor) override { refresh(); XImage *img { XGetImage( xdisplay.get(), @@ -151,12 +145,27 @@ struct x11_attr_t : public display_t { AllPlanes, ZPixmap) }; - if(!cursor) { - return std::make_unique((std::uint8_t*)img->data, img->width, img->height, img); + auto img_out = (x11_img_t*)img_out_base.get(); + img_out->width = img->width; + img_out->height = img->height; + img_out->data = (uint8_t*)img->data; + img_out->img.reset(img); + + if(cursor) { + blend_cursor(xdisplay.get(), (std::uint8_t*)img->data, img->width, img->height); } - blend_cursor(xdisplay.get(), (std::uint8_t*)img->data, img->width, img->height); - return std::make_unique((std::uint8_t*)img->data, img->width, img->height, img); + return capture_e::ok; + } + + int reinit() override { + refresh(); + + return 0; + } + + std::unique_ptr alloc_img() override { + return std::make_unique(); } xdisplay_t xdisplay; @@ -178,25 +187,20 @@ struct shm_attr_t : public x11_attr_t { void delayed_refresh() { refresh(); - refresh_task_id = task_pool.pushDelayed(&shm_attr_t::delayed_refresh, 1s, this).task_id; + refresh_task_id = task_pool.pushDelayed(&shm_attr_t::delayed_refresh, 2s, this).task_id; } shm_attr_t() : x11_attr_t(), shm_xdisplay {XOpenDisplay(nullptr) } { - refresh_task_id = task_pool.pushDelayed(&shm_attr_t::delayed_refresh, 1s, this).task_id; + refresh_task_id = task_pool.pushDelayed(&shm_attr_t::delayed_refresh, 2s, this).task_id; } - ~shm_attr_t() { + ~shm_attr_t() override { while(!task_pool.cancel(refresh_task_id)); } - std::unique_ptr snapshot(bool cursor) override { + capture_e snapshot(std::unique_ptr &img, bool cursor) override { if(display->width_in_pixels != xattr.width || display->height_in_pixels != xattr.height) { - deinit(); - if(init()) { - std::cout << "FATAL ERROR: Couldn't reinitialize shm_attr_t"sv << std::endl; - - std::abort(); - } + return capture_e::reinit; } auto img_cookie = xcb_shm_get_image_unchecked( @@ -213,29 +217,32 @@ struct shm_attr_t : public x11_attr_t { xcb_img_t img_reply { xcb_shm_get_image_reply(xcb.get(), img_cookie, nullptr) }; if(!img_reply) { std::cout << "Info: Could not get image reply"sv << std::endl; - return nullptr; + return capture_e::reinit; } - auto img = std::make_unique(); img->data = new std::uint8_t[frame_size()]; img->width = display->width_in_pixels; img->height = display->height_in_pixels; - std::copy((std::uint8_t*)data.data, (std::uint8_t*)data.data + frame_size(), img->data); + std::copy_n((std::uint8_t*)data.data, frame_size(), img->data); - if(!cursor) { - return img; + if(cursor) { + blend_cursor(shm_xdisplay.get(), img->data, img->width, img->height); } - blend_cursor(shm_xdisplay.get(), img->data, img->width, img->height); - - return img; + return capture_e::ok; } - void deinit() { + std::unique_ptr alloc_img() override { + return std::make_unique(); + } + + int reinit() override { data.~shm_data_t(); shm_id.~shm_id_t(); xcb.reset(nullptr); + + return init(); } int init() { @@ -307,7 +314,7 @@ std::unique_ptr shm_display() { return shm; } -std::unique_ptr display() { +std::shared_ptr display() { auto shm_disp = shm_display(); if(!shm_disp) { diff --git a/sunshine/platform/linux_evdev.cpp b/sunshine/platform/linux_evdev.cpp index 935bf715..eb88d2ec 100644 --- a/sunshine/platform/linux_evdev.cpp +++ b/sunshine/platform/linux_evdev.cpp @@ -426,10 +426,10 @@ input_t input() { std::filesystem::path mouse_path { "sunshine_mouse" }; std::filesystem::path gamepad_path { "sunshine_gamepad" }; - if(std::filesystem::exists(mouse_path)) { + if(std::filesystem::is_symlink(mouse_path)) { std::filesystem::remove(mouse_path); } - if(std::filesystem::exists(gamepad_path)) { + if(std::filesystem::is_symlink(gamepad_path)) { std::filesystem::remove(gamepad_path); } diff --git a/sunshine/platform/windows.cpp b/sunshine/platform/windows.cpp index 0ccf6d13..4bcb3405 100644 --- a/sunshine/platform/windows.cpp +++ b/sunshine/platform/windows.cpp @@ -28,27 +28,292 @@ using output_t = util::safe_ptr>; using output1_t = util::safe_ptr>; using dup_t = util::safe_ptr>; using texture2d_t = util::safe_ptr>; +using resource_t = util::safe_ptr>; extern const char *format_str[]; -struct texture_t { - enum state_e { - UNUSED, - PENDING_MAP, - MAPPED - } state; +class duplication_t { +public: + dup_t dup; + bool has_frame {}; - texture2d_t tex; + capture_e next_frame(DXGI_OUTDUPL_FRAME_INFO &frame_info, resource_t::pointer *res_p) { + HRESULT status; + if(has_frame) { + status = dup->ReleaseFrame(); + + switch(status) { + case S_OK: + has_frame = false; + break; + case DXGI_ERROR_WAIT_TIMEOUT: + return capture_e::timeout; + case WAIT_ABANDONED: + case DXGI_ERROR_ACCESS_DENIED: + return capture_e::reinit; + default: + std::cout << "Error: Couldn't release frame [0x"sv << util::hex(status).to_string_view() << std::endl; + return capture_e::error; + } + } + + status = dup->AcquireNextFrame(1000, &frame_info, res_p); + + switch(status) { + case S_OK: + has_frame = true; + return capture_e::ok; + case DXGI_ERROR_WAIT_TIMEOUT: + return capture_e::timeout; + case WAIT_ABANDONED: + case DXGI_ERROR_ACCESS_DENIED: + return capture_e::reinit; + default: + std::cout << "Error: Couldn't acquire next frame [0x"sv << util::hex(status).to_string_view() << std::endl; + return capture_e::error; + } + } + + capture_e reset(dup_t::pointer dup_p = dup_t::pointer()) { + auto capture_status = capture_e::ok; + if(has_frame) { + capture_status = release_frame(); + } + + dup.reset(dup_p); + + return capture_status; + } + + capture_e release_frame() { + auto status = dup->ReleaseFrame(); + switch (status) { + case S_OK: + return capture_e::ok; + case DXGI_ERROR_WAIT_TIMEOUT: + return capture_e::timeout; + case WAIT_ABANDONED: + case DXGI_ERROR_ACCESS_DENIED: + return capture_e::reinit; + default: + std::cout << "Error: Couldn't release frame [0x"sv << util::hex(status).to_string_view() << std::endl; + return capture_e::error; + } + } + + ~duplication_t() { + if(has_frame) { + release_frame(); + } + } }; -class display_t : public ::platf::display_t { +class display_t; +struct img_t : public ::platf::img_t { + std::shared_ptr owner; //Ensure texture remains valid while img is still needed + + texture2d_t texture; + D3D11_MAPPED_SUBRESOURCE map {}; + + ~img_t() override; +}; + +struct cursor_t { + int x, y; + int width, height; + int pitch; + + std::vector img_data; + bool visible; + +}; + +class display_t : public ::platf::display_t, public std::enable_shared_from_this { public: - std::unique_ptr snapshot(bool cursor) override { - return nullptr; + capture_e snapshot(std::unique_ptr<::platf::img_t> &img_base, bool cursor_visible) override { + auto img = (img_t *) img_base.get(); + HRESULT status; + + DXGI_OUTDUPL_FRAME_INFO frame_info; + + resource_t::pointer res_p {}; + auto capture_status = dup.next_frame(frame_info, &res_p); + resource_t res{res_p}; + + if (capture_status != capture_e::ok) { + return capture_status; + } + + if (frame_info.PointerShapeBufferSize > 0) { + cursor.img_data.resize(frame_info.PointerShapeBufferSize); + DXGI_OUTDUPL_POINTER_SHAPE_INFO shape_info; + + auto &img_data = cursor.img_data; + + UINT dummy; + status = dup.dup->GetFramePointerShape(img_data.size(), img_data.data(), &dummy, &shape_info); + if (FAILED(status)) { + std::cout << "Error: Failed to get new pointer shape [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + + return capture_e::error; + } + + if (shape_info.Type != DXGI_OUTDUPL_POINTER_SHAPE_TYPE_COLOR) { + std::cout << "Warning: Unsupported cursor format ["sv << shape_info.Type << ']' << std::endl; + } + + cursor.width = shape_info.Width; + cursor.height = shape_info.Height; + cursor.pitch = shape_info.Pitch; + } + + if (frame_info.LastPresentTime.QuadPart == 0) { + return capture_e::timeout; + } + + { + texture2d_t::pointer src_p {}; + status = res->QueryInterface(IID_ID3D11Texture2D, (void **)&src_p); + texture2d_t src{src_p}; + + if (FAILED(status)) { + std::cout << "Error: Couldn't query interface [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + return capture_e::error; + } + + //Copy from GPU to CPU + device_ctx->CopyResource((ID3D11Resource *) img->texture.get(), (ID3D11Resource *) src.get()); + } + + cursor.x = frame_info.PointerPosition.Position.x; + cursor.y = frame_info.PointerPosition.Position.y; + cursor.visible = frame_info.PointerPosition.Visible; + + status = device_ctx->Map((ID3D11Resource *) img->texture.get(), 0, D3D11_MAP_READ, 0, &img->map); + if (FAILED(status)) { + if (status == DXGI_ERROR_WAS_STILL_DRAWING) { + return capture_e::timeout; + } + + std::cout << "Error: Failed to map texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + + return capture_e::error; + } + + img->data = (std::uint8_t *) img->map.pData; + img->width = width; + img->height = height; + + return capture_e::ok; + } + + /* + * Called when access is lost. Dup must be reinitialized + */ + int reinit() override { + HRESULT status; + + dup.reset(); + //TODO: Use IDXGIOutput5 for improved performance + { + dxgi::output1_t::pointer output1_p {}; + status = output->QueryInterface(IID_IDXGIOutput1, (void**)&output1_p); + dxgi::output1_t output1 {output1_p }; + + if(FAILED(status)) { + std::cout << "Error: Failed to query IDXGIOutput1 from the output"sv << std::endl; + return -1; + } + + // We try this twice, in case we still get an error on reinitialization + for(int x = 0; x < 2; ++x) { + dxgi::dup_t::pointer dup_p {}; + status = output1->DuplicateOutput((IUnknown*)device.get(), &dup_p); + if(SUCCEEDED(status)) { + dup.reset(dup_p); + break; + } + std::this_thread::sleep_for(200ms); + } + + if(FAILED(status)) { + std::cout << "Error: DuplicateOutput Failed [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + return -1; + } + } + + DXGI_OUTDUPL_DESC dup_desc; + dup.dup->GetDesc(&dup_desc); + + format = dup_desc.ModeDesc.Format; + + std::cout << "Source format ["sv << format_str[dup_desc.ModeDesc.Format] << ']' << std::endl; + + D3D11_TEXTURE2D_DESC t {}; + t.Width = width; + t.Height = height; + t.MipLevels = 1; + t.ArraySize = 1; + t.SampleDesc.Count = 1; + t.Usage = D3D11_USAGE_STAGING; + t.Format = format; + t.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + dxgi::texture2d_t::pointer tex_p {}; + status = device->CreateTexture2D(&t, nullptr, &tex_p); + dxgi::texture2d_t tex { tex_p }; + + if(FAILED(status)) { + std::cout << "Error: Failed to create texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + return -1; + } + + // map the texture simply to get the pitch and stride + D3D11_MAPPED_SUBRESOURCE mapping; + + status = device_ctx->Map((ID3D11Resource *)tex.get(), 0, D3D11_MAP_READ, 0, &mapping); + if(FAILED(status)) { + std::cout << "Error: Failed to map the texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + return -1; + } + + pitch = (int)mapping.RowPitch; + stride = (int)mapping.RowPitch / 4; + + device_ctx->Unmap((ID3D11Resource *)tex.get(), 0); + + return 0; + } + + std::unique_ptr<::platf::img_t> alloc_img() override { + auto img = std::make_unique(); + + D3D11_TEXTURE2D_DESC t {}; + t.Width = width; + t.Height = height; + t.MipLevels = 1; + t.ArraySize = 1; + t.SampleDesc.Count = 1; + t.Usage = D3D11_USAGE_STAGING; + t.Format = format; + t.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + dxgi::texture2d_t::pointer tex_p {}; + HRESULT status = device->CreateTexture2D(&t, nullptr, &tex_p); + dxgi::texture2d_t tex { tex_p }; + + if(FAILED(status)) { + std::cout << "Error: Failed to create texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; + return nullptr; + } + + img->texture = std::move(tex); + img->owner = shared_from_this(); + return img; } int init() { - /* Uncomment when use of IDXGIOutput5 is implemented +/* Uncomment when use of IDXGIOutput5 is implemented std::call_once(windows_cpp_once_flag, []() { DECLARE_HANDLE(DPI_AWARENESS_CONTEXT); const auto DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = ((DPI_AWARENESS_CONTEXT)-4); @@ -81,9 +346,9 @@ public: for(int x = 0; factory_p->EnumAdapters1(x, &adapter_p) != DXGI_ERROR_NOT_FOUND; ++x) { - dxgi::adapter_t adapter { adapter_p }; + dxgi::adapter_t adapter_tmp { adapter_p }; - for(int y = 0; adapter->EnumOutputs(y, &output_p) != DXGI_ERROR_NOT_FOUND; ++y) { + for(int y = 0; adapter_tmp->EnumOutputs(y, &output_p) != DXGI_ERROR_NOT_FOUND; ++y) { dxgi::output_t output_tmp {output_p }; DXGI_OUTPUT_DESC desc; @@ -98,7 +363,7 @@ public: } if(output) { - adapter = std::move(adapter); + adapter = std::move(adapter_tmp); break; } } @@ -162,7 +427,7 @@ public: // Bump up thread priority { - dxgi::dxgi_t::pointer dxgi_p; + dxgi::dxgi_t::pointer dxgi_p {}; status = device->QueryInterface(IID_IDXGIDevice, (void**)&dxgi_p); dxgi::dxgi_t dxgi { dxgi_p }; @@ -176,7 +441,7 @@ public: // Try to reduce latency { - dxgi::dxgi1_t::pointer dxgi_p; + dxgi::dxgi1_t::pointer dxgi_p {}; status = device->QueryInterface(IID_IDXGIDevice, (void**)&dxgi_p); dxgi::dxgi1_t dxgi { dxgi_p }; @@ -188,88 +453,18 @@ public: dxgi->SetMaximumFrameLatency(1); } - //TODO: Use IDXGIOutput5 for improved performance - { - dxgi::output1_t::pointer output1_p {}; - status = output->QueryInterface(IID_IDXGIOutput1, (void**)&output1_p); - dxgi::output1_t output1 {output1_p }; - - if(FAILED(status)) { - std::cout << "Error: Failed to query IDXGIOutput1 from the output"sv << std::endl; - return -1; - } - - // We try this twice, in case we still get an error on reinitialization - for(int x = 0; x < 2; ++x) { - dxgi::dup_t::pointer dup_p {}; - status = output1->DuplicateOutput((IUnknown*)device.get(), &dup_p); - if(SUCCEEDED(status)) { - dup.reset(dup_p); - break; - } - std::this_thread::sleep_for(200ms); - } - - if(FAILED(status)) { - std::cout << "Error: DuplicateOutput Failed [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; - return -1; - } - } - - DXGI_OUTDUPL_DESC dup_desc; - dup->GetDesc(&dup_desc); - - std::cout << "Source format ["sv << format_str[dup_desc.ModeDesc.Format] << ']' << std::endl; - - D3D11_TEXTURE2D_DESC t {}; - t.Width = width; - t.Height = height; - t.MipLevels = 1; - t.ArraySize = 1; - t.SampleDesc.Count = 1; - t.Usage = D3D11_USAGE_STAGING; - t.Format = dup_desc.ModeDesc.Format; - t.CPUAccessFlags = D3D11_CPU_ACCESS_READ; - - textures.resize(3); - for(auto &texture : textures) { - dxgi::texture2d_t::pointer tex_p {}; - status = device->CreateTexture2D(&t, nullptr, &tex_p); - texture.tex.reset(tex_p); - - if(FAILED(status)) { - std::cout << "Error: Failed to create texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; - return -1; - } - } - - // map the texture simply to get the pitch and stride - D3D11_MAPPED_SUBRESOURCE mapping; - - status = device_ctx->Map((ID3D11Resource *)textures[0].tex.get(), 0, D3D11_MAP_READ, 0, &mapping); - if(FAILED(status)) { - std::cout << "Error: Failed to map the texture [0x"sv << util::hex(status).to_string_view() << ']' << std::endl; - return -1; - } - - pitch = (int)mapping.RowPitch; - stride = (int)mapping.RowPitch / 4; - - return 0; + return reinit(); } int deinit() { - for(auto &texture : textures) { - texture.state = texture_t::UNUSED; - texture.tex.reset(); - } - dup.reset(); device_ctx.reset(); device.reset(); output.reset(); adapter.reset(); factory.reset(); + + return 0; } factory1_t factory; @@ -277,15 +472,22 @@ public: output_t output; device_t device; device_ctx_t device_ctx; - dup_t dup; - - std::vector textures; + duplication_t dup; + cursor_t cursor; int width, height; int pitch, stride; + DXGI_FORMAT format; D3D_FEATURE_LEVEL feature_level; }; + +img_t::~img_t() { + if(map.pData) { + owner->device_ctx->Unmap((ID3D11Resource *)texture.get(), 0); + map.pData = nullptr; + } +} } class dummy_mic_t : public mic_t { @@ -303,7 +505,7 @@ std::unique_ptr microphone() { } //std::once_flag windows_cpp_once_flag; -std::unique_ptr display() { +std::shared_ptr display() { auto disp = std::make_unique(); if(disp->init()) { diff --git a/sunshine/stream.cpp b/sunshine/stream.cpp index 6042612b..b9fa4224 100644 --- a/sunshine/stream.cpp +++ b/sunshine/stream.cpp @@ -1019,6 +1019,9 @@ void rtpThread() { server.iterate(config::stream.ping_timeout); if(session.video_packets && !session.video_packets->running()) { + // Ensure all threads are stopping + stop(session); + std::cout << "Waiting for Audio to end..."sv << std::endl; session.audioThread.join(); std::cout << "Waiting for Video to end..."sv << std::endl; diff --git a/sunshine/video.cpp b/sunshine/video.cpp index 7a381a42..bc1eb2e0 100644 --- a/sunshine/video.cpp +++ b/sunshine/video.cpp @@ -188,24 +188,40 @@ void capture_display(packet_queue_t packets, idr_event_t idr_events, config_t co int framerate = config.framerate; - img_event_t images {new img_event_t::element_type }; - - std::thread encoderThread { &encodeThread, images, packets, idr_events, config }; - auto disp = platf::display(); + if(!disp) { + packets->stop(); + return; + } + + img_event_t images {new img_event_t::element_type }; + std::thread encoderThread { &encodeThread, images, packets, idr_events, config }; auto time_span = std::chrono::floor(1s) / framerate; while(packets->running()) { auto next_snapshot = std::chrono::steady_clock::now() + time_span; - auto img = disp->snapshot(display_cursor); - if(!img) { - std::this_thread::sleep_until(next_snapshot); - continue; + auto img = disp->alloc_img(); + auto status = disp->snapshot(img, display_cursor); + + switch(status) { + case platf::capture_e::reinit: + if(disp->reinit()) { + packets->stop(); + } + continue; + case platf::capture_e::timeout: + std::this_thread::sleep_until(next_snapshot); + continue; + case platf::capture_e::error: + packets->stop(); + continue; + // Prevent warning during compilation + case platf::capture_e::ok: + break; } images->raise(std::move(img)); - img.reset(); auto t = std::chrono::steady_clock::now(); if(t > next_snapshot) {