diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index d96e10485c29e..4a4e7adcbf388 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -1281,9 +1281,7 @@ impl Step for Sysroot {
             }
 
             // Copy the compiler into the correct sysroot.
-            let ci_rustc_dir =
-                builder.config.out.join(&*builder.config.build.triple).join("ci-rustc");
-            builder.cp_r(&ci_rustc_dir, &sysroot);
+            builder.cp_r(&builder.ci_rustc_dir(builder.build.build), &sysroot);
             return INTERNER.intern_path(sysroot);
         }
 
@@ -1385,7 +1383,10 @@ impl Step for Assemble {
 
         // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
         if builder.download_rustc() {
-            builder.ensure(Sysroot { compiler: target_compiler });
+            let sysroot = builder.ensure(Sysroot { compiler: target_compiler });
+            // Ensure that `libLLVM.so` ends up in the newly created target directory,
+            // so that tools using `rustc_private` can use it.
+            dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot);
             return target_compiler;
         }
 
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 94e71b89b5c86..9498b772f58e0 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -1962,6 +1962,20 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
         }
     }
 
+    // FIXME: for reasons I don't understand, the LLVM so in the `rustc` component is different than the one in `rust-dev`.
+    // Only the one in `rustc` works with the downloaded compiler.
+    if builder.download_rustc() && target == builder.build.build {
+        let src_libdir = builder.ci_rustc_dir(target).join("lib");
+        for entry in t!(std::fs::read_dir(&src_libdir)) {
+            let entry = t!(entry);
+            if entry.file_name().to_str().unwrap().starts_with("libLLVM-") {
+                install_llvm_file(builder, &entry.path(), dst_libdir);
+                return !builder.config.dry_run();
+            }
+        }
+        panic!("libLLVM.so not found in src_libdir {}!", src_libdir.display());
+    }
+
     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
     // clear why this is the case, though. llvm-config will emit the versioned
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 1ecb52e75f190..419bcbc63cfff 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -814,6 +814,11 @@ impl Build {
         self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir())
     }
 
+    /// Directory where the extracted `rustc-dev` component is stored.
+    fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf {
+        self.out.join(&*target.triple).join("ci-rustc")
+    }
+
     /// Root output directory for LLVM compiled for `target`
     ///
     /// Note that if LLVM is configured externally then the directory returned