Support skipping SIP patch in layer config (#3269)

* Support skipping SIP patch in layer config

* Fix cargo fmt

* Use list in test config

* Add CI action step to build issue3248 test app

* Add changelog

* Correct config schema JSON

* Set test run log level to trace

* Assert DYLD_* env var is not in test app stderr

* Print SIP status from runner

* Codesign script and binary before running tests

* Fix clippy

* Check no patched files in temp dir

* Fix cargo deny

* Wrap patch and skip binaries in an option type

Author: 0x00A5

.github/workflows/ci.yaml

@@ -366,6 +366,9 @@ jobs:
       - run: |
           cd mirrord/layer/tests/apps/issue2438
           cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue3248
+          cargo build
       - run: |
           cd mirrord/layer/tests/apps/rebind0
           cargo build
@@ -414,6 +417,8 @@ jobs:
         uses: metalbear-co/setup-protoc@3ea1d70ac22caff0b66ed6cb37d5b7aadebd4623
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: verify SIP status
+        run: csrutil status
       # this is required for clippy lint because the layer path variable is checked at compile time
       - name: create fake layer file
         run: touch /tmp/file.dylib
@@ -494,6 +499,9 @@ jobs:
       - run: |
           cd mirrord/layer/tests/apps/issue2438
           cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue3248
+          cargo build
       - run: |
           cd mirrord/layer/tests/apps/rebind0
           cargo build

Cargo.lock

@@ -14,6 +14,7 @@ members = [
     "mirrord/layer/tests/apps/issue1899",
     "mirrord/layer/tests/apps/issue2001",
     "mirrord/layer/tests/apps/issue2438",
+    "mirrord/layer/tests/apps/issue3248",
     "mirrord/layer/tests/apps/rebind0",
     "sample/rust",
     "medschool",

Cargo.toml

@@ -0,0 +1 @@
+Added the option to skip sip patching.

changelog.d/+skip-sip.added.md

@@ -142,6 +142,18 @@
         }
       ]
     },
+    "skip_sip": {
+      "title": "skip_sip {#root-skip_sip}",
+      "description": "Allows mirrord to skip patching (macOS SIP) unwanted processes.\n\nWhen patching is skipped, mirrord will no longer be able to load into the process and its child processes.",
+      "anyOf": [
+        {
+          "$ref": "#/definitions/VecOrSingle_for_String"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
     "target": {
       "title": "target {#root-target}",
       "anyOf": [

mirrord-schema.json

@@ -17,7 +17,7 @@ use mirrord_protocol::{
     LogLevel,
 };
 #[cfg(target_os = "macos")]
-use mirrord_sip::sip_patch;
+use mirrord_sip::{sip_patch, SipPatchOptions};
 use mirrord_tls_util::SecureChannelSetup;
 use semver::Version;
 use serde::Serialize;
@@ -308,11 +308,18 @@ impl MirrordExecution {
             .and_then(|exe| {
                 sip_patch(
                     exe,
-                    &config
-                        .sip_binaries
-                        .clone()
-                        .map(|x| x.to_vec())
-                        .unwrap_or_default(),
+                    SipPatchOptions {
+                        patch: &config
+                            .sip_binaries
+                            .clone()
+                            .map(|x| x.to_vec())
+                            .unwrap_or_default(),
+                        skip: &config
+                            .skip_sip
+                            .clone()
+                            .map(|x| x.to_vec())
+                            .unwrap_or_default(),
+                    },
                 )
                 .transpose() // We transpose twice to propagate a possible error out of this
                              // closure.

mirrord/cli/src/execution.rs

@@ -32,7 +32,7 @@
 //! the [`LayerConfig`], and checks it with [`LayerConfig::verify`] (which is similar to what's done
 //! in the `mirrord verify-config` command).
 //!
-//! - Tip: [`Progress`] logging might be incovenient sometimes when you want to see normal Rust
+//! - Tip: [`Progress`] logging might be inconvenient sometimes when you want to see normal Rust
 //!   logs, you can disable it with the `MIRRORD_PROGRESS_MODE=off` env var.
 //!
 //! Next, we start the target resolution, and how the target is resolved depends if the

mirrord/cli/src/main.rs

@@ -329,6 +329,15 @@ pub struct LayerConfig {
     /// ## experimental {#root-experimental}
     #[config(nested)]
     pub experimental: ExperimentalConfig,
+
+    /// ## skip_sip {#root-skip_sip}
+    ///
+    /// Allows mirrord to skip patching (macOS SIP) unwanted processes.
+    ///
+    /// When patching is skipped, mirrord will no longer be able to load into
+    /// the process and its child processes.
+    #[config(env = "MIRRORD_SKIP_SIP")]
+    pub skip_sip: Option<VecOrSingle<String>>,
 }
 
 impl LayerConfig {
@@ -1002,6 +1011,7 @@ mod tests {
             internal_proxy: None,
             use_proxy: None,
             experimental: None,
+            skip_sip: None,
         };
 
         assert_eq!(config, expect);

mirrord/config/src/lib.rs

@@ -65,5 +65,8 @@ tests = { path = "../../tests" }
 test-cdylib = "1"
 tokio = { workspace = true, features = ["rt", "net", "macros", "fs"] }
 
+[target.'cfg(target_os = "macos")'.dev-dependencies]
+apple-codesign = { version = "0.29", default-features = false }
+
 [lib]
 crate-type = ["cdylib"]

mirrord/layer/Cargo.toml

@@ -7,7 +7,7 @@ use std::{
 
 use libc::{c_char, c_int, pid_t};
 use mirrord_layer_macro::{hook_fn, hook_guard_fn};
-use mirrord_sip::{sip_patch, SipError, MIRRORD_PATCH_DIR};
+use mirrord_sip::{sip_patch, SipError, SipPatchOptions, MIRRORD_PATCH_DIR};
 use null_terminated::Nul;
 use tracing::{trace, warn};
 
@@ -33,13 +33,19 @@ const MAX_ARGC: usize = 256;
 
 pub(crate) static PATCH_BINARIES: OnceLock<Vec<String>> = OnceLock::new();
 
+pub(crate) static SKIP_PATCH_BINARIES: OnceLock<Vec<String>> = OnceLock::new();
+
 pub(crate) unsafe fn enable_macos_hooks(
     hook_manager: &mut HookManager,
     patch_binaries: Vec<String>,
+    skip_binaries: Vec<String>,
 ) {
     PATCH_BINARIES
         .set(patch_binaries)
         .expect("couldn't set patch_binaries");
+    SKIP_PATCH_BINARIES
+        .set(skip_binaries)
+        .expect("couldn't set skip_binaries");
     replace!(
         hook_manager,
         "posix_spawn",
@@ -61,7 +67,16 @@ pub(crate) unsafe fn enable_macos_hooks(
 #[mirrord_layer_macro::instrument(level = "trace")]
 pub(super) fn patch_if_sip(path: &str) -> Detour<String> {
     let patch_binaries = PATCH_BINARIES.get().expect("patch binaries not set");
-    match sip_patch(path, patch_binaries) {
+    let skip_patch_binaries = SKIP_PATCH_BINARIES
+        .get()
+        .expect("skip patch binaries not set");
+    match sip_patch(
+        path,
+        SipPatchOptions {
+            patch: patch_binaries,
+            skip: skip_patch_binaries,
+        },
+    ) {
         Ok(None) => Bypass(NoSipDetected(path.to_string())),
         Ok(Some(new_path)) => Success(new_path),
         Err(SipError::FileNotFound(non_existing_bin)) => {

mirrord/layer/src/exec_utils.rs

@@ -107,6 +107,8 @@ use crate::{
 #[cfg(test)]
 mod integration_tests_deps {
     use actix_codec as _;
+    #[cfg(target_os = "macos")]
+    use apple_codesign as _;
     use futures as _;
     use mirrord_intproxy as _;
     use serde_json as _;
@@ -197,14 +199,27 @@ fn layer_pre_initialization() -> Result<(), LayerError> {
         .map(|x| x.to_vec())
         .unwrap_or_default();
 
+    #[cfg(target_os = "macos")]
+    let skip_patch_binaries = config
+        .skip_sip
+        .clone()
+        .map(|x| x.to_vec())
+        .unwrap_or_default();
+
     // SIP Patch the process' binary then re-execute it. Needed
     // for https://github.com/metalbear-co/mirrord/issues/1529
     #[cfg(target_os = "macos")]
     if given_process.ends_with("dotnet") {
         let path = EXECUTABLE_PATH
             .get()
             .expect("EXECUTABLE_PATH needs to be set!");
-        if let Ok(Some(binary)) = mirrord_sip::sip_patch(path, &patch_binaries) {
+        if let Ok(Some(binary)) = mirrord_sip::sip_patch(
+            path,
+            mirrord_sip::SipPatchOptions {
+                patch: &patch_binaries,
+                skip: &skip_patch_binaries,
+            },
+        ) {
             let err = exec::execvp(
                 binary,
                 EXECUTABLE_ARGS
@@ -221,7 +236,7 @@ fn layer_pre_initialization() -> Result<(), LayerError> {
     match given_process.load_type(&config) {
         LoadType::Full => layer_start(config),
         #[cfg(target_os = "macos")]
-        LoadType::SIPOnly => sip_only_layer_start(config, patch_binaries),
+        LoadType::SIPOnly => sip_only_layer_start(config, patch_binaries, skip_patch_binaries),
         LoadType::Skip => load_only_layer_start(&config),
     }
 
@@ -488,14 +503,20 @@ fn fetch_env_vars() -> HashMap<String, String> {
 /// We need to hook execve syscall to allow mirrord-layer to be loaded with sip patch when loading
 /// mirrord-layer on a process where specified to skip with MIRRORD_SKIP_PROCESSES
 #[cfg(target_os = "macos")]
-fn sip_only_layer_start(mut config: LayerConfig, patch_binaries: Vec<String>) {
+fn sip_only_layer_start(
+    mut config: LayerConfig,
+    patch_binaries: Vec<String>,
+    skip_patch_binaries: Vec<String>,
+) {
     use mirrord_config::feature::fs::READONLY_FILE_BUFFER_DEFAULT;
 
     load_only_layer_start(&config);
 
     let mut hook_manager = HookManager::default();
 
-    unsafe { exec_utils::enable_macos_hooks(&mut hook_manager, patch_binaries) };
+    unsafe {
+        exec_utils::enable_macos_hooks(&mut hook_manager, patch_binaries, skip_patch_binaries)
+    };
     unsafe { exec_hooks::hooks::enable_exec_hooks(&mut hook_manager) };
     // we need to hook file access to patch path to our temp bin.
     config.feature.fs = FsConfig {
@@ -591,7 +612,8 @@ fn enable_hooks(state: &LayerSetup) {
         use crate::exec_utils::enable_macos_hooks;
 
         let patch_binaries = state.sip_binaries();
-        unsafe { enable_macos_hooks(&mut hook_manager, patch_binaries) };
+        let skip_patch_binaries = state.skip_patch_binaries();
+        unsafe { enable_macos_hooks(&mut hook_manager, patch_binaries, skip_patch_binaries) };
 
         if state.experimental().trust_any_certificate {
             unsafe { tls::enable_tls_hooks(&mut hook_manager) };

mirrord/layer/src/lib.rs

@@ -151,6 +151,15 @@ impl LayerSetup {
             .unwrap_or_default()
     }
 
+    #[cfg(target_os = "macos")]
+    pub fn skip_patch_binaries(&self) -> Vec<String> {
+        self.config
+            .skip_sip
+            .as_deref()
+            .map(<[_]>::to_vec)
+            .unwrap_or_default()
+    }
+
     pub fn is_debugger_port(&self, addr: &SocketAddr) -> bool {
         self.debugger_ports.contains(addr)
     }

mirrord/layer/src/setup.rs

@@ -0,0 +1,7 @@
+[package]
+name = "issue3248"
+version = "0.1.0"
+edition = "2021"
+license.workspace = true
+
+[dependencies]

mirrord/layer/tests/apps/issue3248/Cargo.toml

@@ -0,0 +1,7 @@
+fn main() {
+    let script_path = std::env::var("TEST_SCRIPT_PATH").unwrap();
+    let binary_path = std::env::var("TEST_BINARY_PATH").unwrap();
+    let _output = std::process::Command::new(script_path).output().unwrap();
+
+    let _output = std::process::Command::new(binary_path).output().unwrap();
+}

mirrord/layer/tests/apps/issue3248/src/main.rs

@@ -9,7 +9,7 @@ use mirrord_protocol::{
     ClientMessage, DaemonMessage, FileRequest, FileResponse,
 };
 #[cfg(target_os = "macos")]
-use mirrord_sip::sip_patch;
+use mirrord_sip::{sip_patch, SipPatchOptions};
 use rstest::rstest;
 use tokio::net::TcpListener;
 
@@ -37,7 +37,9 @@ async fn bash_script(dylib_path: &Path, config_dir: &Path) {
     println!("Listening for messages from the layer on {addr}");
     let env = get_env(dylib_path, addr, vec![], Some(&config_path));
     #[cfg(target_os = "macos")]
-    let executable = sip_patch(&executable, &Vec::new()).unwrap().unwrap();
+    let executable = sip_patch(&executable, SipPatchOptions::default())
+        .unwrap()
+        .unwrap();
     let test_process = TestProcess::start_process(executable, application.get_args(), env).await;
 
     let mut intproxy = TestIntProxy::new(listener).await;

mirrord/layer/tests/bash.rs

@@ -25,7 +25,7 @@ use mirrord_protocol::{
     ClientMessage, DaemonCodec, DaemonMessage, FileRequest, FileResponse,
 };
 #[cfg(target_os = "macos")]
-use mirrord_sip::sip_patch;
+use mirrord_sip::{sip_patch, SipPatchOptions};
 use rstest::fixture;
 pub use tests::utils::process::TestProcess;
 use tokio::{
@@ -843,6 +843,7 @@ pub enum Application {
     NodeIssue2283,
     RustIssue2204,
     RustIssue2438,
+    RustIssue3248,
     NodeIssue2807,
     RustRebind0,
     /// Go application that simply opens a file.
@@ -1009,6 +1010,13 @@ impl Application {
                     "../../target/debug/issue2001"
                 )
             }
+            Application::RustIssue3248 => {
+                format!(
+                    "{}/{}",
+                    env!("CARGO_MANIFEST_DIR"),
+                    "../../target/debug/issue3248"
+                )
+            }
             Application::RustRebind0 => {
                 format!(
                     "{}/{}",
@@ -1165,6 +1173,7 @@ impl Application {
             | Application::RustIssue2204
             | Application::RustRebind0
             | Application::RustIssue2438
+            | Application::RustIssue3248
             | Application::Go23Issue2988 => vec![],
             Application::RustOutgoingUdp => ["--udp", RUST_OUTGOING_LOCAL, RUST_OUTGOING_PEERS]
                 .into_iter()
@@ -1257,6 +1266,7 @@ impl Application {
             | Application::NodeIssue2283
             | Application::RustIssue2204
             | Application::RustIssue2438
+            | Application::RustIssue3248
             | Application::NodeIssue2807
             | Application::RustRebind0
             | Application::Go23Open { .. }
@@ -1272,7 +1282,7 @@ impl Application {
     pub async fn get_test_process(&self, env: HashMap<String, String>) -> TestProcess {
         let executable = self.get_executable().await;
         #[cfg(target_os = "macos")]
-        let executable = sip_patch(&executable, &Vec::new())
+        let executable = sip_patch(&executable, SipPatchOptions::default())
             .unwrap()
             .unwrap_or(executable);
         println!("Using executable: {}", &executable);