fix: wayland controlled side, cursor misalignment (#13537)

Signed-off-by: fufesou <linlong1266@gmail.com>
This commit is contained in:
fufesou
2025-11-18 00:37:15 +08:00
committed by GitHub
parent a6571e71e4
commit b2dff336ce
22 changed files with 1241 additions and 187 deletions

View File

@@ -10,7 +10,7 @@ authors = ["Ram <quadrupleslap@gmail.com>"]
edition = "2018"
[features]
wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"]
wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing", "zbus"]
mediacodec = ["ndk"]
linux-pkg-config = ["dep:pkg-config"]
hwcodec = ["dep:hwcodec"]
@@ -57,6 +57,7 @@ tracing = { version = "0.1", optional = true }
gstreamer = { version = "0.16", optional = true }
gstreamer-app = { version = "0.16", features = ["v1_10"], optional = true }
gstreamer-video = { version = "0.16", optional = true }
zbus = { version = "3.15", optional = true }
[dependencies.hwcodec]
git = "https://github.com/rustdesk-org/hwcodec"

View File

@@ -88,6 +88,27 @@ impl Display {
}
}
pub fn scale(&self) -> f64 {
match self {
Display::X11(_d) => 1.0,
Display::WAYLAND(d) => d.scale(),
}
}
pub fn logical_width(&self) -> usize {
match self {
Display::X11(d) => d.width(),
Display::WAYLAND(d) => d.logical_width(),
}
}
pub fn logical_height(&self) -> usize {
match self {
Display::X11(d) => d.height(),
Display::WAYLAND(d) => d.logical_height(),
}
}
pub fn origin(&self) -> (i32, i32) {
match self {
Display::X11(d) => d.origin(),

View File

@@ -8,7 +8,6 @@ use super::x11::PixelBuffer;
pub struct Capturer(Display, Box<dyn Recorder>, Vec<u8>);
lazy_static::lazy_static! {
static ref MAP_ERR: RwLock<Option<fn(err: String)-> io::Error>> = Default::default();
}
@@ -61,7 +60,7 @@ impl TraitCapturer for Capturer {
}
}
pub struct Display(pipewire::PipeWireCapturable);
pub struct Display(pub(crate) pipewire::PipeWireCapturable);
impl Display {
pub fn primary() -> io::Result<Display> {
@@ -81,11 +80,35 @@ impl Display {
}
pub fn width(&self) -> usize {
self.0.size.0
self.physical_width()
}
pub fn height(&self) -> usize {
self.0.size.1
self.physical_height()
}
pub fn physical_width(&self) -> usize {
self.0.physical_size.0
}
pub fn physical_height(&self) -> usize {
self.0.physical_size.1
}
pub fn logical_width(&self) -> usize {
self.0.logical_size.0
}
pub fn logical_height(&self) -> usize {
self.0.logical_size.1
}
pub fn scale(&self) -> f64 {
if self.logical_width() == 0 {
1.0
} else {
self.physical_width() as f64 / self.logical_width() as f64
}
}
pub fn origin(&self) -> (i32, i32) {
@@ -97,7 +120,7 @@ impl Display {
}
pub fn is_primary(&self) -> bool {
false
self.0.primary
}
pub fn name(&self) -> String {

View File

@@ -1,5 +1,6 @@
pub mod capturable;
pub mod pipewire;
pub mod display;
mod screencast_portal;
mod request_portal;
pub mod remote_desktop_portal;

View File

@@ -0,0 +1,256 @@
use hbb_common::regex::Regex;
use lazy_static::lazy_static;
use std::sync::Mutex;
use std::{
process::{Command, Output, Stdio},
sync::Arc,
time::{Duration, Instant},
};
use tracing::warn;
use hbb_common::platform::linux::{get_wayland_displays, WaylandDisplayInfo};
lazy_static! {
static ref DISPLAYS: Mutex<Option<Arc<Displays>>> = Mutex::new(None);
}
const COMMAND_TIMEOUT: Duration = Duration::from_millis(1000);
pub struct Displays {
pub primary: usize,
pub displays: Vec<WaylandDisplayInfo>,
}
// We need this helper to run commands with a timeout, as some commands may hang.
// `kscreen-doctor -o` is known to hang when:
// 1. On Archlinux, Both GNOME and KDE Plasma are installed.
// 2. Run this command in a GNOME session.
fn run_with_timeout(
program: &str,
args: &[&str],
timeout: Duration,
label: &str,
) -> Option<Output> {
let mut child = Command::new(program)
.args(args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.ok()?;
let start = Instant::now();
loop {
if let Ok(Some(_)) = child.try_wait() {
break;
}
if start.elapsed() >= timeout {
warn!("{} command timed out after {:?}", label, timeout);
if let Err(e) = child.kill() {
warn!("Failed to kill child process for '{}': {}", label, e);
}
if let Err(e) = child.wait() {
warn!("Failed to wait for child process for '{}': {}", label, e);
}
return None;
}
std::thread::sleep(Duration::from_millis(30));
}
match child.wait_with_output() {
Ok(output) => {
if !output.status.success() {
warn!("{} command failed with status: {}", label, output.status);
return None;
}
Some(output)
}
Err(_) => None,
}
}
// There are some limitations with xrandr method:
// 1. It only works when XWayland is running.
// 2. The distro may not have xrandr installed by default.
// 3. xrandr may not report "primary" in its output. eg. openSUSE Leap 15.6 KDE Plasma.
fn try_xrandr_primary() -> Option<String> {
let output = Command::new("xrandr").output().ok()?;
if !output.status.success() {
return None;
}
let text = String::from_utf8_lossy(&output.stdout);
for line in text.lines() {
if line.contains("primary") && line.contains("connected") {
if let Some(name) = line.split_whitespace().next() {
return Some(name.to_string());
}
}
}
None
}
fn try_kscreen_primary() -> Option<String> {
if !hbb_common::platform::linux::is_kde_session() {
return None;
}
let output = run_with_timeout(
"kscreen-doctor",
&["-o"],
COMMAND_TIMEOUT,
"kscreen-doctor -o",
)?;
if !output.status.success() {
return None;
}
let text = String::from_utf8_lossy(&output.stdout);
// Remove ANSI color codes
let re_ansi = Regex::new(r"\x1b\[[0-9;]*m").ok()?;
let clean_text = re_ansi.replace_all(&text, "");
// Split the text into blocks, each starting with "Output:".
// The first element of the split will be empty, so we skip it.
for block in clean_text.split("Output:").skip(1) {
// Check if this block describes the primary monitor.
if block.contains("priority 1") {
// The monitor name is the second piece of text in the block, after the ID.
// e.g., " 1 eDP-1 enabled..." -> "eDP-1"
if let Some(name) = block.split_whitespace().nth(1) {
return Some(name.to_string());
}
}
}
None
}
fn try_gdbus_primary() -> Option<String> {
let output = run_with_timeout(
"gdbus",
&[
"call",
"--session",
"--dest",
"org.gnome.Mutter.DisplayConfig",
"--object-path",
"/org/gnome/Mutter/DisplayConfig",
"--method",
"org.gnome.Mutter.DisplayConfig.GetCurrentState",
],
COMMAND_TIMEOUT,
"gdbus DisplayConfig.GetCurrentState",
)?;
if !output.status.success() {
return None;
}
let text = String::from_utf8_lossy(&output.stdout);
// Match logical monitor entries with primary=true
// Pattern: (x, y, scale, transform, true, [('connector-name', ...), ...], ...)
// Use regex to find entries where 5th field is true, then extract connector name
// Example matched text: "(0, 0, 1.5, 0, true, [('HDMI-1', 'MHH', 'Monitor', '0x00000000')], ...)"
let re = Regex::new(r"\([^()]*,\s*true,\s*\[\('([^']+)'").ok()?;
if let Some(captures) = re.captures(&text) {
return captures.get(1).map(|m| m.as_str().to_string());
}
None
}
fn get_primary_monitor() -> Option<String> {
try_xrandr_primary()
.or_else(try_kscreen_primary)
.or_else(try_gdbus_primary)
}
pub fn get_displays() -> Arc<Displays> {
let mut lock = DISPLAYS.lock().unwrap();
match lock.as_ref() {
Some(displays) => displays.clone(),
None => match get_wayland_displays() {
Ok(displays) => {
let mut primary_index = None;
if let Some(name) = get_primary_monitor() {
for (i, display) in displays.iter().enumerate() {
if display.name == name {
primary_index = Some(i);
break;
}
}
};
if primary_index.is_none() {
for (i, display) in displays.iter().enumerate() {
if display.x == 0 && display.y == 0 {
primary_index = Some(i);
break;
}
}
}
let displays = Arc::new(Displays {
primary: primary_index.unwrap_or(0),
displays,
});
*lock = Some(displays.clone());
displays
}
Err(err) => {
warn!("Failed to get wayland displays: {}", err);
Arc::new(Displays {
primary: 0,
displays: Vec::new(),
})
}
},
}
}
#[inline]
pub fn clear_wayland_displays_cache() {
let _ = DISPLAYS.lock().unwrap().take();
}
// Return (min_x, max_x, min_y, max_y)
pub fn get_desktop_rect_for_uinput() -> Option<(i32, i32, i32, i32)> {
let wayland_displays = get_displays();
let displays = &wayland_displays.displays;
if displays.is_empty() {
return None;
}
// For compatibility, if only one display, we use the physical size for `uinput`.
// Otherwise, we use the logical size for `uinput`.
if displays.len() == 1 {
let d = &displays[0];
return Some((d.x, d.x + d.width, d.y, d.y + d.height));
}
let mut min_x = i32::MAX;
let mut min_y = i32::MAX;
let mut max_x = i32::MIN;
let mut max_y = i32::MIN;
for d in displays.iter() {
min_x = min_x.min(d.x);
min_y = min_y.min(d.y);
let size = if let Some(logical_size) = d.logical_size {
logical_size
} else {
// When `logical_size` is None, we cannot obtain the correct desktop rectangle.
// This may occur if the Wayland compositor does not provide logical size information,
// or if display information is incomplete. We fall back to physical size, which provides
// usable dimensions, but may not always be correct depending on compositor behavior.
warn!(
"Display at ({}, {}) is missing logical_size; falling back to physical size ({}, {}).",
d.x, d.y, d.width, d.height
);
(d.width, d.height)
};
max_x = max_x.max(d.x + size.0);
max_y = max_y.max(d.y + size.1);
}
Some((min_x, max_x, min_y, max_y))
}

View File

@@ -2,9 +2,12 @@ use std::collections::HashMap;
use std::error::Error;
use std::os::unix::io::AsRawFd;
use std::process::Command;
use std::sync::{atomic::AtomicBool, Arc, Mutex};
use std::sync::{
atomic::{AtomicBool, AtomicU8, Ordering},
Arc, Mutex,
};
use std::time::Duration;
use tracing::{debug, trace, warn};
use tracing::{debug, error, trace, warn};
use dbus::{
arg::{OwnedFd, PropMap, RefArg, Variant},
@@ -17,23 +20,63 @@ use gstreamer as gst;
use gstreamer::prelude::*;
use gstreamer_app::AppSink;
use hbb_common::config;
use lazy_static::lazy_static;
use hbb_common::{bail, config, platform::linux::CMD_SH, tokio, ResultType};
use super::capturable::PixelProvider;
use super::capturable::{Capturable, Recorder};
use super::display::{clear_wayland_displays_cache, get_displays, Displays};
use super::remote_desktop_portal::OrgFreedesktopPortalRemoteDesktop as remote_desktop_portal;
use super::request_portal::OrgFreedesktopPortalRequestResponse;
use super::screencast_portal::OrgFreedesktopPortalScreenCast as screencast_portal;
use hbb_common::platform::linux::CMD_SH;
use lazy_static::lazy_static;
lazy_static! {
pub static ref RDP_SESSION_INFO: Mutex<Option<RdpSessionInfo>> = Mutex::new(None);
// Maybe it's better to save this cache in config file?
// Because "--server" process may be restarted frequently, then the cache will be lost.
// But the users have to know where to find and delete the config file when they want to clear the cache,
// or we have to add a UI for that.
// For simplicity, we just keep it in memory for now.
static ref PIPEWIRE_DISPLAY_OFFSET_CACHE: Mutex<Option<PipewireDisplayOffsetCache>> =
Mutex::new(None);
}
// For KDE Plasma only, because GNOME provides position info.
struct PipewireDisplayOffsetCache {
// We need to compare the displays, because:
// 1. On Archlinux KDE Plasma
// 2. One display, and connect, remember share choice.
// 3. Plug in another monitor.
// 4. The portal will reuse the restore token, no new share choice dialog, but the share screen is different.
// The controlling side will see the new monitor.
// All displays as one string for easy comparison
// name1-x1-y1-width1-height1;name2-x2-y2-width2-height2;...
display_key: String,
restore_token: String,
offsets: Vec<(i32, i32)>,
}
// KDE Plasma may not provide position info
static HAS_POSITION_ATTR: AtomicBool = AtomicBool::new(false);
static IS_SERVER_RUNNING: AtomicU8 = AtomicU8::new(0); // 0: uninitialized, 1:true, 2: false
impl PipewireDisplayOffsetCache {
fn displays_to_key(displays: &Arc<Displays>) -> String {
displays
.displays
.iter()
.map(|d| format!("{}-{}-{}-{}-{}", d.name, d.x, d.y, d.width, d.height))
.collect::<Vec<String>>()
.join(";")
}
}
#[inline]
pub fn close_session() {
let _ = RDP_SESSION_INFO.lock().unwrap().take();
clear_wayland_displays_cache();
HAS_POSITION_ATTR.store(false, Ordering::SeqCst);
}
#[inline]
@@ -52,6 +95,8 @@ pub fn try_close_session() {
}
if close {
*rdp_info = None;
clear_wayland_displays_cache();
HAS_POSITION_ATTR.store(false, Ordering::SeqCst);
}
}
@@ -75,6 +120,10 @@ impl PwStreamInfo {
pub fn get_size(&self) -> (usize, usize) {
self.size
}
pub fn get_position(&self) -> (i32, i32) {
self.position
}
}
#[derive(Debug)]
@@ -108,8 +157,10 @@ pub struct PipeWireCapturable {
fd: OwnedFd,
path: u64,
source_type: u64,
pub primary: bool,
pub position: (i32, i32),
pub size: (usize, usize),
pub logical_size: (usize, usize),
pub physical_size: (usize, usize),
}
impl PipeWireCapturable {
@@ -117,27 +168,31 @@ impl PipeWireCapturable {
conn: Arc<SyncConnection>,
fd: OwnedFd,
resolution: Arc<Mutex<Option<(usize, usize)>>>,
stream: PwStreamInfo,
stream: &PwStreamInfo,
) -> Self {
// alternative to get screen resolution as stream.size is not always correct ex: on fractional scaling
// https://github.com/rustdesk/rustdesk/issues/6116#issuecomment-1817724244
let size = get_res(Self {
let physical_size = get_res(Self {
dbus_conn: conn.clone(),
fd: fd.clone(),
path: stream.path,
source_type: stream.source_type,
primary: false,
position: stream.position,
size: stream.size,
logical_size: stream.size,
physical_size: (0, 0),
})
.unwrap_or(stream.size);
*resolution.lock().unwrap() = Some(size);
*resolution.lock().unwrap() = Some(physical_size);
Self {
dbus_conn: conn,
fd,
path: stream.path,
source_type: stream.source_type,
primary: false,
position: stream.position,
size,
logical_size: stream.size,
physical_size,
}
}
}
@@ -214,7 +269,7 @@ pub struct PipeWireRecorder {
}
impl PipeWireRecorder {
pub fn new(capturable: PipeWireCapturable) -> Result<Self, Box<dyn Error>> {
pub fn new(capturable: PipeWireCapturable) -> ResultType<Self> {
let pipeline = gst::Pipeline::new(None);
let src = gst::ElementFactory::make("pipewiresrc", None)?;
@@ -247,7 +302,36 @@ impl PipeWireRecorder {
));
appsink.set_caps(Some(&caps));
// [Workaround]
// Crash may occur if there are multiple pipelines started at the same time.
// `pipeline.get_state()` can significantly reduce the probability of crashes,
// but cannot completely resolve this issue.
// Adding a short sleep period can also reduce the probability of crashes.
debug!(
"[gstreamer] Setting pipeline {} to PLAYING state...",
capturable.fd.as_raw_fd()
);
pipeline.set_state(gst::State::Playing)?;
// Wait for the state change to actually complete before proceeding.
// The 2000ms timeout for pipeline state change was chosen based on empirical testing.
let state_change = pipeline.get_state(gst::ClockTime::from_mseconds(2000));
match state_change {
(Ok(_), gst::State::Playing, _) => {
debug!(
"[gstreamer] Pipeline {} state confirmed as PLAYING.",
capturable.fd.as_raw_fd()
);
}
(result, state, pending) => {
warn!(
"[gstreamer] Pipeline {} state change incomplete: result={:?}, state={:?}, pending={:?}",
capturable.fd.as_raw_fd(), result, state, pending
);
}
}
std::thread::sleep(std::time::Duration::from_millis(150));
Ok(Self {
pipeline,
appsink,
@@ -366,6 +450,8 @@ impl Drop for PipeWireRecorder {
if let Err(err) = self.pipeline.set_state(gst::State::Null) {
warn!("Failed to stop GStreamer pipeline: {}.", err);
}
// Wait for state change to complete to avoid races during PipeWire teardown.
let _ = self.pipeline.get_state(gst::ClockTime::from_mseconds(2000));
}
}
@@ -396,18 +482,18 @@ where
0 => {}
1 => {
warn!("DBus response: User cancelled interaction.");
failure_out.store(true, std::sync::atomic::Ordering::Relaxed);
failure_out.store(true, Ordering::SeqCst);
return true;
}
c => {
warn!("DBus response: Unknown error, code: {}.", c);
failure_out.store(true, std::sync::atomic::Ordering::Relaxed);
failure_out.store(true, Ordering::SeqCst);
return true;
}
}
if let Err(err) = f(r, c, m) {
warn!("Error requesting screen capture via dbus: {}", err);
failure_out.store(true, std::sync::atomic::Ordering::Relaxed);
failure_out.store(true, Ordering::SeqCst);
}
true
})
@@ -488,6 +574,7 @@ fn streams_from_response(response: OrgFreedesktopPortalRequestResponse) -> Vec<P
if v.len() == 2 {
info.position.0 = v[0] as _;
info.position.1 = v[1] as _;
HAS_POSITION_ATTR.store(true, Ordering::SeqCst);
}
}
}
@@ -510,16 +597,15 @@ pub fn get_available_cursor_modes() -> Result<u32, dbus::Error> {
}
// mostly inspired by https://gitlab.gnome.org/-/snippets/39
pub fn request_remote_desktop() -> Result<
(
SyncConnection,
OwnedFd,
Vec<PwStreamInfo>,
dbus::Path<'static>,
bool,
),
Box<dyn Error>,
> {
pub fn request_remote_desktop(
capture_cursor: bool,
) -> ResultType<(
SyncConnection,
OwnedFd,
Vec<PwStreamInfo>,
dbus::Path<'static>,
bool,
)> {
unsafe {
if !INIT {
gstreamer::init()?;
@@ -574,6 +660,7 @@ pub fn request_remote_desktop() -> Result<
session.clone(),
failure.clone(),
is_support_restore_token,
capture_cursor,
),
failure_res.clone(),
)?;
@@ -586,7 +673,7 @@ pub fn request_remote_desktop() -> Result<
break;
}
if failure_res.load(std::sync::atomic::Ordering::Relaxed) {
if failure_res.load(Ordering::SeqCst) {
break;
}
}
@@ -607,9 +694,7 @@ pub fn request_remote_desktop() -> Result<
}
}
}
Err(Box::new(DBusError(
"Failed to obtain screen capture. You may need to upgrade the PipeWire library for better compatibility. Please check https://github.com/rustdesk/rustdesk/issues/8600#issuecomment-2254720954 for more details.".into()
)))
bail!("Failed to obtain screen capture. You may need to upgrade the PipeWire library for better compatibility. Please check https://github.com/rustdesk/rustdesk/issues/8600#issuecomment-2254720954 for more details.")
}
fn on_create_session_response(
@@ -618,6 +703,7 @@ fn on_create_session_response(
session: Arc<Mutex<Option<dbus::Path<'static>>>>,
failure: Arc<AtomicBool>,
is_support_restore_token: bool,
capture_cursor: bool,
) -> impl Fn(
OrgFreedesktopPortalRequestResponse,
&SyncConnection,
@@ -666,6 +752,14 @@ fn on_create_session_response(
}
args.insert("types".into(), Variant(Box::new(1u32))); //| 2u32)));
if capture_cursor {
get_available_cursor_modes().ok().map(|modes| {
if modes & 0x2 != 0 {
args.insert("cursor_mode".to_string(), Variant(Box::new(2u32)));
}
});
}
let path = portal.select_sources(ses.clone(), args)?;
handle_response(
c,
@@ -838,7 +932,7 @@ pub fn get_capturables() -> Result<Vec<PipeWireCapturable>, Box<dyn Error>> {
};
if rdp_connection.is_none() {
let (conn, fd, streams, session, is_support_restore_token) = request_remote_desktop()?;
let (conn, fd, streams, session, is_support_restore_token) = request_remote_desktop(false)?;
let conn = Arc::new(conn);
let rdp_info = RdpSessionInfo {
@@ -852,7 +946,7 @@ pub fn get_capturables() -> Result<Vec<PipeWireCapturable>, Box<dyn Error>> {
*rdp_connection = Some(rdp_info);
}
let rdp_info = match rdp_connection.as_ref() {
let rdp_info = match rdp_connection.as_mut() {
Some(res) => res,
None => {
return Err(Box::new(DBusError("RDP response is None.".into())));
@@ -861,8 +955,7 @@ pub fn get_capturables() -> Result<Vec<PipeWireCapturable>, Box<dyn Error>> {
Ok(rdp_info
.streams
.clone()
.into_iter()
.iter()
.map(|s| {
PipeWireCapturable::new(
rdp_info.conn.clone(),
@@ -883,7 +976,12 @@ pub fn get_capturables() -> Result<Vec<PipeWireCapturable>, Box<dyn Error>> {
//
// `screencast_portal` supports restore_token and persist_mode if the version is greater than or equal to 4.
// `remote_desktop_portal` does not support restore_token and persist_mode.
fn is_server_running() -> bool {
pub(crate) fn is_server_running() -> bool {
let v = IS_SERVER_RUNNING.load(Ordering::SeqCst);
if v > 0 {
return v == 1;
}
let app_name = config::APP_NAME.read().unwrap().clone().to_lowercase();
let output = match Command::new(CMD_SH.as_str())
.arg("-c")
@@ -898,5 +996,525 @@ fn is_server_running() -> bool {
let output_str = String::from_utf8_lossy(&output.stdout);
let is_running = output_str.contains(&format!("{} --server", app_name));
IS_SERVER_RUNNING.store(if is_running { 1 } else { 2 }, Ordering::SeqCst);
is_running
}
// The logical size reported by portal may be different from the size reported by `get_displays()`.
// So we need to use the workaround here.
// 1. openSUSE, KDE Plasma
// 2. Kubuntu 24.04 TLS, after running `sudo apt install plasma-workspace-wayland`
// Maybe it's a bug, and we can remove this workaround in the future.
pub fn try_fix_logical_size(shared_displays: &mut Vec<crate::Display>) {
if !is_server_running() {
return;
}
let wayland_displays = get_displays();
if wayland_displays.displays.is_empty() {
return;
}
for sd in shared_displays.iter_mut() {
if let crate::Display::WAYLAND(d) = sd {
let capturable = &mut d.0;
for wd in wayland_displays.displays.iter() {
if capturable.position.0 == wd.x && capturable.position.1 == wd.y {
if let Some(logical_size) = wd.logical_size {
if capturable.physical_size.0 != wd.width as usize
|| capturable.physical_size.1 != wd.height as usize
{
// If "Full Workspace" is selected in the portal dialog,
// the physical size reported by portal may not match the display info.
debug!(
"Physical size of capturable ({:?}) does not match display info: ({:?}) - ({:?}). Skipping logical size fix.",
capturable.position,
capturable.physical_size,
(wd.width as usize, wd.height as usize)
);
break;
}
if capturable.logical_size.0 != logical_size.0 as usize
|| capturable.logical_size.1 != logical_size.1 as usize
{
warn!(
"Fixing logical size of capturable from {:?} to {:?} based on display info {:?}.",
capturable.logical_size,
logical_size,
wd
);
capturable.logical_size =
(logical_size.0 as usize, logical_size.1 as usize);
}
}
break;
}
}
}
}
}
pub fn fill_displays(
mouse_move_to: impl Fn(i32, i32),
get_cursor_pos: fn() -> Option<(i32, i32)>,
shared_displays: &mut Vec<crate::Display>,
) -> ResultType<()> {
if !is_server_running() {
return Ok(());
}
let mut rdp_connection = RDP_SESSION_INFO.lock().unwrap();
let rdp_info = match rdp_connection.as_mut() {
Some(res) => res,
None => {
// Unreachable
bail!("RDP session info is None when filling display positions.");
}
};
let all_displays = get_displays();
if !HAS_POSITION_ATTR.load(Ordering::SeqCst) {
if all_displays.displays.len() > 1 {
debug!("Multiple Wayland displays detected, adjusting stream positions accordingly.");
try_fill_positions(
mouse_move_to,
get_cursor_pos,
&all_displays,
shared_displays,
&mut rdp_info.streams,
)?;
}
HAS_POSITION_ATTR.store(true, Ordering::SeqCst);
}
if all_displays.displays.len() > 1 {
sort_streams(&all_displays, shared_displays, &mut rdp_info.streams);
}
shared_displays.iter_mut().next().map(|d| {
if let crate::Display::WAYLAND(d) = d {
d.0.primary = true;
}
});
Ok(())
}
fn try_fill_positions(
mouse_move_to: impl Fn(i32, i32),
get_cursor_pos: fn() -> Option<(i32, i32)>,
displays: &Arc<Displays>,
shared_displays: &mut Vec<crate::Display>,
streams: &mut Vec<PwStreamInfo>,
) -> ResultType<()> {
if try_fill_positions_from_cache(displays, shared_displays, streams) {
return Ok(());
}
let mut multi_matched_indices = Vec::new();
for (i, sd) in shared_displays.iter_mut().enumerate() {
if let crate::Display::WAYLAND(d) = sd {
let capturable = &mut d.0;
let mut match_count = 0;
for wd in displays.displays.iter() {
if capturable.physical_size.0 == wd.width as usize
&& capturable.physical_size.1 == wd.height as usize
{
capturable.position = (wd.x, wd.y);
if let Some(pw_stream) = streams.get_mut(i) {
pw_stream.position = (wd.x, wd.y);
}
match_count += 1;
}
}
if match_count == 0 {
warn!(
"No matching display found for capturable with size {:?}.",
capturable.physical_size
);
} else if match_count > 1 {
multi_matched_indices.push(i);
}
}
}
if !multi_matched_indices.is_empty() {
fill_multi_matched_positions(
mouse_move_to,
get_cursor_pos,
displays,
shared_displays,
streams,
multi_matched_indices,
)?;
}
save_positions_to_cache(displays, shared_displays);
Ok(())
}
fn try_fill_positions_from_cache(
displays: &Arc<Displays>,
shared_displays: &mut Vec<crate::Display>,
streams: &mut Vec<PwStreamInfo>,
) -> bool {
let mut lock = PIPEWIRE_DISPLAY_OFFSET_CACHE.lock().unwrap();
let Some(cache) = lock.as_ref() else {
return false;
};
if cache.offsets.len() != shared_displays.len() {
let _ = lock.take();
return false;
}
let display_key = PipewireDisplayOffsetCache::displays_to_key(displays);
if cache.display_key != display_key {
let _ = lock.take();
return false;
}
let restore_token = config::LocalConfig::get_option(RESTORE_TOKEN_CONF_KEY);
if cache.restore_token != restore_token {
let _ = lock.take();
return false;
}
for (i, sd) in shared_displays.iter_mut().enumerate() {
if let crate::Display::WAYLAND(d) = sd {
let capturable = &mut d.0;
if let Some((x_off, y_off)) = cache.offsets.get(i) {
capturable.position = (*x_off, *y_off);
if let Some(pw_stream) = streams.get_mut(i) {
pw_stream.position = (*x_off, *y_off);
}
}
}
}
true
}
fn save_positions_to_cache(displays: &Arc<Displays>, shared_displays: &Vec<crate::Display>) {
let restore_token = config::LocalConfig::get_option(RESTORE_TOKEN_CONF_KEY);
if restore_token.is_empty() {
return;
}
let mut offsets = Vec::new();
for sd in shared_displays.iter() {
if let crate::Display::WAYLAND(d) = sd {
let capturable = &d.0;
offsets.push((capturable.position.0, capturable.position.1));
}
}
let display_key = PipewireDisplayOffsetCache::displays_to_key(displays);
let cache = PipewireDisplayOffsetCache {
display_key,
restore_token,
offsets,
};
*PIPEWIRE_DISPLAY_OFFSET_CACHE.lock().unwrap() = Some(cache);
}
fn compare_left_up_corner(w: usize, d1: &[u8], d2: &[u8]) -> bool {
if w == 0 {
return false;
}
if d1.len() != d2.len() {
return false;
}
let bpp = 4; // BGR0/RGB0
let stride = w.saturating_mul(bpp);
if stride == 0 || d1.len() < stride || d2.len() < stride {
return false;
}
let h = d1.len() / stride;
if h == 0 {
return false;
}
let roi_w = std::cmp::min(36, w);
let roi_h = std::cmp::min(36, h);
let mut diff_px = 0usize;
let total_px = roi_w * roi_h;
// Minimum number of differing pixels required to consider images different.
const MIN_DIFF_PIXELS: usize = 8;
// Divisor for threshold calculation: allows up to 1/8 of ROI pixels to differ before returning true.
const DIFF_THRESHOLD_DIVISOR: usize = 8;
let threshold = std::cmp::max(MIN_DIFF_PIXELS, total_px / DIFF_THRESHOLD_DIVISOR);
for y in 0..roi_h {
let row_off = y * stride;
for x in 0..roi_w {
let i = row_off + x * bpp;
let a = &d1[i..i + bpp];
let b = &d2[i..i + bpp];
if a != b {
diff_px += 1;
if diff_px >= threshold {
return true;
}
}
}
}
false
}
fn fill_multi_matched_positions(
mouse_move_to: impl Fn(i32, i32),
get_cursor_pos: fn() -> Option<(i32, i32)>,
displays: &Arc<Displays>,
shared_displays: &mut Vec<crate::Display>,
streams: &mut Vec<PwStreamInfo>,
multi_matched_indices: Vec<usize>,
) -> ResultType<()> {
debug!(
"Multiple capturables ({:?}) match the same display size, attempting to disambiguate positions.",
&multi_matched_indices);
if multi_matched_indices.is_empty() {
return Ok(());
}
let is_support_embeded_cursor = get_available_cursor_modes()
.ok()
.map(|modes| modes & 0x2 != 0)
.unwrap_or(false);
if is_support_embeded_cursor {
fill_multi_matched_positions_cursor(
mouse_move_to,
get_cursor_pos,
displays,
shared_displays,
streams,
multi_matched_indices,
)?;
}
Ok(())
}
fn mouse_move_to_(
mouse_move_to: &impl Fn(i32, i32),
get_cursor_pos: fn() -> Option<(i32, i32)>,
x: i32,
y: i32,
) {
const MOVE_MOUSE_TIMEOUT: Duration = Duration::from_millis(150);
let start = std::time::Instant::now();
while start.elapsed() < MOVE_MOUSE_TIMEOUT {
mouse_move_to(x, y);
std::thread::sleep(Duration::from_millis(20));
if let Some((x1, y1)) = get_cursor_pos() {
if x1 == x && y1 == y {
return;
}
}
}
warn!(
"Failed to move mouse to ({}, {}) within timeout: {:?}.",
x, y, &MOVE_MOUSE_TIMEOUT
);
}
fn fill_multi_matched_positions_cursor(
mouse_move_to: impl Fn(i32, i32),
get_cursor_pos: fn() -> Option<(i32, i32)>,
displays: &Arc<Displays>,
shared_displays: &mut Vec<crate::Display>,
streams: &mut Vec<PwStreamInfo>,
multi_matched_indices: Vec<usize>,
) -> ResultType<()> {
// This creates a new remote desktop session for cursor-based position detection.
// The session is temporary, used only for disambiguation, and is dropped after detection completes.
let (conn, fd, streams_with_cursor, _session, _is_support_restore_token) =
request_remote_desktop(true)?;
let conn = Arc::new(conn);
let mut matched_indices = Vec::new();
const CAPTURE_TIMEOUT_MS: u64 = 1_000;
for idx in multi_matched_indices {
match (
shared_displays.get_mut(idx),
streams.get_mut(idx),
streams_with_cursor.get(idx),
) {
(Some(crate::Display::WAYLAND(d)), Some(pw_stream), Some(pw_stream_with_cursor)) => {
// Check if only one display matches the size
let mut match_count = 0;
for (i, wd) in displays.displays.iter().enumerate() {
if matched_indices.contains(&i) {
continue;
}
if d.0.physical_size.0 == wd.width as usize
&& d.0.physical_size.1 == wd.height as usize
{
match_count += 1;
}
}
if match_count == 0 {
error!(
"No matching display found for capturable with size {:?}.",
d.0.physical_size
);
continue;
}
if match_count == 1 {
for (i, wd) in displays.displays.iter().enumerate() {
if matched_indices.contains(&i) {
continue;
}
if d.0.physical_size.0 == wd.width as usize
&& d.0.physical_size.1 == wd.height as usize
{
d.0.position = (wd.x, wd.y);
pw_stream.position = (wd.x, wd.y);
matched_indices.push(i);
debug!(
"Disambiguated position for capturable with size {:?} to ({}, {}).",
d.0.physical_size, wd.x, wd.y
);
break;
}
}
continue;
}
// Move the mouse to a neutral position first,
// to avoid interference from previous position.
mouse_move_to_(&mouse_move_to, get_cursor_pos, 300, 300);
let mut rec = PipeWireRecorder::new(PipeWireCapturable {
dbus_conn: conn.clone(),
fd: fd.clone(),
path: pw_stream_with_cursor.path,
source_type: pw_stream_with_cursor.source_type,
primary: false,
position: pw_stream_with_cursor.position,
logical_size: pw_stream_with_cursor.size,
physical_size: (0, 0),
})?;
// Take first frame and copy owned buffer to avoid borrow across second capture
let (is_bgr, w, first_buf): (bool, usize, Vec<u8>) =
match rec.capture(CAPTURE_TIMEOUT_MS) {
Ok(PixelProvider::BGR0(w, _, data1)) => (true, w, data1.to_vec()),
Ok(PixelProvider::RGB0(w, _, data1)) => (false, w, data1.to_vec()),
Ok(_) => {
error!("Unexpected pixel format on first capture.");
continue;
}
Err(e) => {
error!(
"Failed to capture screen for position disambiguation: {}",
e
);
continue;
}
};
let matched_len = matched_indices.len();
for (i, wd) in displays.displays.iter().enumerate() {
if matched_indices.contains(&i) {
continue;
}
if wd.width as usize == d.0.physical_size.0
&& wd.height as usize == d.0.physical_size.1
{
mouse_move_to_(&mouse_move_to, get_cursor_pos, wd.x + 8, wd.y + 8);
rec.saved_raw_data.clear();
match rec.capture(CAPTURE_TIMEOUT_MS) {
Ok(PixelProvider::BGR0(_, _, data2)) if is_bgr => {
if compare_left_up_corner(w, &first_buf, data2) {
d.0.position = (wd.x, wd.y);
pw_stream.position = (wd.x, wd.y);
matched_indices.push(i);
debug!(
"Disambiguated position for capturable with size {:?} to ({}, {}).",
d.0.physical_size, wd.x, wd.y
);
break;
}
}
Ok(PixelProvider::RGB0(_, _, data2)) if !is_bgr => {
if compare_left_up_corner(w, &first_buf, data2) {
d.0.position = (wd.x, wd.y);
pw_stream.position = (wd.x, wd.y);
matched_indices.push(i);
debug!(
"Disambiguated position for capturable with size {:?} to ({}, {}).",
d.0.physical_size, wd.x, wd.y
);
break;
}
}
Ok(_) => {
// unreachable
error!("Pixel format changed between captures, cannot disambiguate position.");
}
Err(e) => {
error!(
"Failed to capture screen for position disambiguation: {}",
e
);
}
}
}
}
if matched_len == matched_indices.len() {
error!(
"Failed to disambiguate position for capturable with size {:?}.",
d.0.physical_size
);
}
}
_ => {}
}
}
Ok(())
}
fn sort_streams(
displays: &Arc<Displays>,
shared_displays: &mut Vec<crate::Display>,
streams: &mut Vec<PwStreamInfo>,
) {
if streams.is_empty() {
// unreachable
error!("No streams available to sort.");
return;
}
// put the main display first, then the rest by the order of displays
let mut display_order: Vec<(i32, i32)> = Vec::new();
if let Some(d) = displays.displays.get(displays.primary) {
display_order.push((d.x, d.y));
}
for (i, d) in displays.displays.iter().enumerate() {
if i != displays.primary {
display_order.push((d.x, d.y));
}
}
let mut sorted_streams = Vec::new();
let mut sorted_shared_displays = Vec::new();
// Move matching items in order without cloning
for (x, y) in display_order.into_iter() {
for i in 0..streams.len() {
if streams[i].position.0 == x && streams[i].position.1 == y {
sorted_streams.push(streams.remove(i));
// shared_displays.len() must be equal to streams.len()
// But we still check the length to avoid panic
if shared_displays.len() > i {
sorted_shared_displays.push(shared_displays.remove(i));
}
break;
}
}
}
*streams = sorted_streams;
*shared_displays = sorted_shared_displays;
}