Skip to content

Commit

Permalink
Make try_mark_previous_green lock-free.
Browse files Browse the repository at this point in the history
  • Loading branch information
@cjgillot
cjgillot committed Feb 24, 2021
1 parent ac230b7 commit 064bbb8
Show file tree
Hide file tree
Showing 3 changed files with 288 additions and 219 deletions.
11 changes: 10 additions & 1 deletion compiler/rustc_data_structures/src/sync.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ cfg_if! {
}
}

use std::ops::Add;
use std::ops::{Add, BitOr};
use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe};

/// This is a single threaded variant of AtomicCell provided by crossbeam.
Expand Down Expand Up @@ -147,6 +147,15 @@ cfg_if! {
}
}

impl<T: BitOr<Output=T> + Copy> Atomic<T> {
#[inline]
pub fn fetch_or(&self, val: T, _: Ordering) -> T {
let old = self.0.get();
self.0.set(old | val);
old
}
}

pub type AtomicUsize = Atomic<usize>;
pub type AtomicBool = Atomic<bool>;
pub type AtomicU32 = Atomic<u32>;
Expand Down
92 changes: 51 additions & 41 deletions compiler/rustc_query_system/src/dep_graph/graph.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ impl std::convert::From<DepNodeIndex> for QueryInvocationId {
}

struct DepGraphData<K: DepKind> {
/// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
serialized: SerializedDepGraph<K>,

/// The dep-graph from the compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
graph: RwLock<CurrentDepGraph<K>>,
Expand Down Expand Up @@ -118,9 +122,11 @@ impl<K: DepKind> DepGraph<K> {
Err(_) => None,
};

let graph = RwLock::new(CurrentDepGraph::new(&prev_graph));
DepGraph {
data: Some(Lrc::new(DepGraphData {
graph: RwLock::new(CurrentDepGraph::new(prev_graph)),
serialized: prev_graph,
graph,
emitting_diagnostics: Default::default(),
previous_work_products: prev_work_products,
dep_node_debug: Default::default(),
Expand Down Expand Up @@ -148,7 +154,7 @@ impl<K: DepKind> DepGraph<K> {

pub fn query(&self) -> DepGraphQuery<K> {
let data = self.data.as_ref().unwrap();
data.graph.read().query()
data.graph.read().query(&data.serialized)
}

pub fn assert_ignored(&self) {
Expand Down Expand Up @@ -239,6 +245,7 @@ impl<K: DepKind> DepGraph<K> {

// Intern the new `DepNode`.
let dep_node_index = data.graph.write().intern_task_node(
&data.serialized,
key,
&edges[..],
current_fingerprint,
Expand Down Expand Up @@ -284,8 +291,11 @@ impl<K: DepKind> DepGraph<K> {
hash: data.anon_id_seed.combine(hasher.finish()).into(),
};

let dep_node_index =
data.graph.write().intern_anon_node(target_dep_node, &task_deps.reads[..]);
let dep_node_index = data.graph.write().intern_anon_node(
&data.serialized,
target_dep_node,
&task_deps.reads[..],
);

(result, dep_node_index)
} else {
Expand Down Expand Up @@ -334,7 +344,10 @@ impl<K: DepKind> DepGraph<K> {
#[cfg(debug_assertions)]
if let Some(target) = task_deps.node {
if let Some(ref forbidden_edge) = _data.forbidden_edge {
let src = self.dep_node_of(dep_node_index);
let src = _data
.graph
.read()
.index_to_node(&_data.serialized, dep_node_index);
if forbidden_edge.test(&src, &target) {
panic!("forbidden edge {:?} -> {:?} created", src, target)
}
Expand All @@ -352,24 +365,18 @@ impl<K: DepKind> DepGraph<K> {
#[inline]
fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
let data = self.data.as_ref()?;
data.graph.read().dep_node_index_of_opt(dep_node)
data.graph.read().dep_node_index_of_opt(&data.serialized, dep_node)
}

#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.dep_node_index_of_opt(dep_node).is_some()
}

#[inline]
fn dep_node_of(&self, dep_node_index: DepNodeIndex) -> DepNode<K> {
let data = self.data.as_ref().unwrap();
data.graph.read().dep_node_of(dep_node_index)
}

#[inline]
pub(crate) fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
let data = self.data.as_ref().unwrap();
data.graph.read().fingerprint_of(dep_node_index)
data.graph.read().fingerprint_of(&data.serialized, dep_node_index)
}

/// Checks whether a previous work product exists for `v` and, if
Expand Down Expand Up @@ -405,8 +412,8 @@ impl<K: DepKind> DepGraph<K> {
pub fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
if let Some(ref data) = self.data {
let graph = data.graph.read();
if let Some(prev_index) = graph.node_to_index_opt(dep_node) {
return graph.color(prev_index);
if let Some(prev_index) = graph.node_to_index_opt(&data.serialized, dep_node) {
return data.serialized.color(prev_index);
} else {
// This is a node that did not exist in the previous compilation
// session, so we consider it to be red.
Expand Down Expand Up @@ -444,8 +451,8 @@ impl<K: DepKind> DepGraph<K> {
let data = self.data.as_ref()?;

// Return None if the dep node didn't exist in the previous session
let prev_index = data.graph.read().node_to_index_opt(dep_node)?;
let prev_deps = data.graph.read().color_or_edges(prev_index);
let prev_index = data.graph.read().node_to_index_opt(&data.serialized, dep_node)?;
let prev_deps = data.serialized.color_or_edges(prev_index);
let prev_deps = match prev_deps {
Err(prev_deps) => prev_deps,
Ok(DepNodeColor::Green) => return Some((prev_index, prev_index.rejuvenate())),
Expand All @@ -468,7 +475,7 @@ impl<K: DepKind> DepGraph<K> {
parent_dep_node_index: SerializedDepNodeIndex,
dep_node: &DepNode<K>,
) -> Option<()> {
let dep_dep_node_color = data.graph.read().color_or_edges(parent_dep_node_index);
let dep_dep_node_color = data.serialized.color_or_edges(parent_dep_node_index);
let prev_deps = match dep_dep_node_color {
Ok(DepNodeColor::Green) => {
// This dependency has been marked as green before, we are
Expand All @@ -477,7 +484,7 @@ impl<K: DepKind> DepGraph<K> {
debug!(
"try_mark_parent_green({:?}) --- found dependency {:?} to be immediately green",
dep_node,
data.graph.read().index_to_node(parent_dep_node_index)
data.serialized.index_to_node(parent_dep_node_index)
);
return Some(());
}
Expand All @@ -489,7 +496,7 @@ impl<K: DepKind> DepGraph<K> {
debug!(
"try_mark_parent_green({:?}) - END - dependency {:?} was immediately red",
dep_node,
data.graph.read().index_to_node(parent_dep_node_index)
data.serialized.index_to_node(parent_dep_node_index)
);
return None;
}
Expand All @@ -503,12 +510,12 @@ impl<K: DepKind> DepGraph<K> {
is unknown, trying to mark it green",
dep_node,
{
let dep_dep_node = data.graph.read().index_to_node(parent_dep_node_index);
let dep_dep_node = data.serialized.index_to_node(parent_dep_node_index);
(dep_dep_node, dep_dep_node.hash)
}
);

let dep_dep_node = &data.graph.read().index_to_node(parent_dep_node_index);
let dep_dep_node = &data.serialized.index_to_node(parent_dep_node_index);
let node_index =
self.try_mark_previous_green(tcx, data, parent_dep_node_index, prev_deps, dep_dep_node);
if node_index.is_some() {
Expand All @@ -533,7 +540,7 @@ impl<K: DepKind> DepGraph<K> {
return None;
}

let dep_dep_node_color = data.graph.read().color(parent_dep_node_index);
let dep_dep_node_color = data.serialized.color(parent_dep_node_index);

match dep_dep_node_color {
Some(DepNodeColor::Green) => {
Expand Down Expand Up @@ -592,15 +599,15 @@ impl<K: DepKind> DepGraph<K> {

// We never try to mark eval_always nodes as green
debug_assert!(!dep_node.kind.is_eval_always());
debug_assert_eq!(data.graph.read().index_to_node(prev_dep_node_index), *dep_node);
debug_assert_eq!(data.serialized.index_to_node(prev_dep_node_index), *dep_node);

for &dep_dep_node_index in prev_deps {
self.try_mark_parent_green(tcx, data, dep_dep_node_index, dep_node)?
}

#[cfg(not(parallel_compiler))]
debug_assert_eq!(
data.graph.read().color(prev_dep_node_index),
data.serialized.color(prev_dep_node_index),
None,
"DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
insertion for {:?}",
Expand All @@ -616,7 +623,7 @@ impl<K: DepKind> DepGraph<K> {
let dep_node_index = {
// We allocating an entry for the node in the current dependency graph and
// adding all the appropriate edges imported from the previous graph
data.graph.write().intern_dark_green_node(prev_dep_node_index)
data.serialized.intern_dark_green_node(prev_dep_node_index)
};

// ... and emitting any stored diagnostic.
Expand Down Expand Up @@ -672,11 +679,10 @@ impl<K: DepKind> DepGraph<K> {
let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion");

let data = self.data.as_ref().unwrap();
let graph = data.graph.read();
for prev_index in graph.serialized_indices() {
match graph.color(prev_index) {
for prev_index in data.serialized.serialized_indices() {
match data.serialized.color(prev_index) {
Some(DepNodeColor::Green) => {
let dep_node = data.graph.read().index_to_node(prev_index);
let dep_node = data.serialized.index_to_node(prev_index);
debug!("PROMOTE {:?} {:?}", prev_index, dep_node);
qcx.try_load_from_on_disk_cache(&dep_node);
}
Expand All @@ -692,11 +698,10 @@ impl<K: DepKind> DepGraph<K> {
// Register reused dep nodes (i.e. nodes we've marked red or green) with the context.
pub fn register_reused_dep_nodes<Ctxt: DepContext<DepKind = K>>(&self, tcx: Ctxt) {
let data = self.data.as_ref().unwrap();
let graph = data.graph.read();
for prev_index in graph.serialized_indices() {
match graph.color(prev_index) {
for prev_index in data.serialized.serialized_indices() {
match data.serialized.color(prev_index) {
Some(_) => {
let dep_node = data.graph.read().index_to_node(prev_index);
let dep_node = data.serialized.index_to_node(prev_index);
tcx.register_reused_dep_node(&dep_node);
}
None => {}
Expand All @@ -717,17 +722,17 @@ impl<K: DepKind> DepGraph<K> {

let mut stats: FxHashMap<_, Stat<K>> = FxHashMap::with_hasher(Default::default());

for index in prev.live_indices() {
let kind = prev.dep_node_of(index).kind;
let edge_count = prev.edge_targets_from(index).len();
for index in prev.live_indices(&data.serialized) {
let kind = prev.index_to_node(&data.serialized, index).kind;
let edge_count = prev.edge_targets_from(&data.serialized, index).len();

let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
}

let total_node_count = prev.node_count();
let total_edge_count = prev.edge_count();
let total_node_count = prev.node_count(&data.serialized);
let total_edge_count = prev.edge_count(&data.serialized);

// Drop the lock guard.
std::mem::drop(prev);
Expand Down Expand Up @@ -792,14 +797,19 @@ impl<K: DepKind> DepGraph<K> {
}

pub fn compression_map(&self) -> IndexVec<DepNodeIndex, Option<SerializedDepNodeIndex>> {
self.data.as_ref().unwrap().graph.read().compression_map()
let data = self.data.as_ref().unwrap();
data.graph.read().compression_map(&data.serialized)
}

pub fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error>
where
K: Encodable<E>,
{
if let Some(data) = &self.data { data.graph.read().encode(encoder) } else { Ok(()) }
if let Some(data) = &self.data {
data.graph.read().encode(&data.serialized, encoder)
} else {
Ok(())
}
}
}

Expand Down
Loading

0 comments on commit 064bbb8

Please sign in to comment.