From 67cb41fc5729ccdbddd6a874017f52a6c5d3dc9f Mon Sep 17 00:00:00 2001 From: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:35:08 +0100 Subject: [PATCH] rustc_codegen_ssa: Make upstream monomorphizations representation sparse Upstream monomorphisations are a blessing and a curse of Zed's build performance. On one hand, we do benefit from it, as builds with share-generics disabled are slower for us (plus we don't really want to use nightly anyways). On the other, deserializing query results takes *a lot* of time. For some crates close to the end of our compilation pipeline, it's over 400ms per crate. To make matters worse, I've measured a hit ratio of upstream generics. A sample of such measurement goes as follows: ``` upstream_monomorphization returned None for 28501 distinct monomorphizations. upstream_monomorphization returned Some 2518 times. Results came from 163 distinct CrateNums. In total, there are 619731 instantiations ``` This is horrid for us, as we're using a very small percentage of the map that we spend so much time deserializing from. This commit tries to (rather clumsily) move us towards a sparse representation of upstream_monomorphizations. Instead of storing )> which is rather heavy to deserialize, we'll resort to storing Hashes of Instances. This commit reduces a `touch crates/editor/src/editor.rs` scenario in Zed for me from 14.5s to 11s. --- Cargo.lock | 1 + .../src/back/symbol_export.rs | 29 +++++++ compiler/rustc_metadata/Cargo.toml | 1 + compiler/rustc_metadata/src/rmeta/decoder.rs | 9 ++ .../src/rmeta/decoder/cstore_impl.rs | 1 + compiler/rustc_metadata/src/rmeta/encoder.rs | 29 +++++++ .../src/rmeta/exported_symbol_hash_map.rs | 87 +++++++++++++++++++ compiler/rustc_metadata/src/rmeta/mod.rs | 3 + .../rustc_metadata/src/rmeta/parameterized.rs | 1 + .../src/middle/exported_symbols.rs | 35 +++++++- compiler/rustc_middle/src/queries.rs | 28 ++++++ compiler/rustc_middle/src/query/keys.rs | 7 ++ compiler/rustc_middle/src/ty/instance.rs | 28 ++++-- 13 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 compiler/rustc_metadata/src/rmeta/exported_symbol_hash_map.rs diff --git a/Cargo.lock b/Cargo.lock index 545c776b4805e..d6b6457f590fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4205,6 +4205,7 @@ dependencies = [ "rustc_expand", "rustc_feature", "rustc_fs_util", + "rustc_hashes", "rustc_hir", "rustc_hir_pretty", "rustc_incremental", diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index c6e9456919215..8983d4ce90bb6 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -2,7 +2,9 @@ use std::collections::hash_map::Entry::*; use rustc_abi::{CanonAbi, X86Call}; use rustc_ast::expand::allocator::{AllocatorKind, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name}; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::unord::UnordMap; +use rustc_hashes::Hash128; use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE, LocalDefId}; use rustc_middle::bug; @@ -472,6 +474,30 @@ fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) !tcx.reachable_set(()).contains(&def_id) } +fn upstream_monomorphization_hashes_provider( + tcx: TyCtxt<'_>, + (): (), +) -> FxIndexMap> { + let mut map: FxIndexMap> = Default::default(); + for &cnum in tcx.crates(()) { + for &hash in tcx.exported_generic_symbol_hashes(cnum) { + map.entry(hash).or_default().push(cnum); + } + } + // Sort each candidate list by StableCrateId for determinism. + for candidates in map.values_mut() { + candidates.sort_by_key(|&cnum| tcx.stable_crate_id(cnum)); + } + map +} + +fn upstream_monomorphization_for_hash_provider( + tcx: TyCtxt<'_>, + hash: Hash128, +) -> Option<&smallvec::SmallVec<[CrateNum; 1]>> { + tcx.upstream_monomorphization_hashes(()).get(&hash) +} + pub(crate) fn provide(providers: &mut Providers) { providers.queries.reachable_non_generics = reachable_non_generics_provider; providers.queries.is_reachable_non_generic = is_reachable_non_generic_provider_local; @@ -481,6 +507,9 @@ pub(crate) fn provide(providers: &mut Providers) { providers.queries.is_unreachable_local_definition = is_unreachable_local_definition_provider; providers.queries.upstream_drop_glue_for = upstream_drop_glue_for_provider; providers.queries.upstream_async_drop_glue_for = upstream_async_drop_glue_for_provider; + providers.queries.upstream_monomorphization_hashes = upstream_monomorphization_hashes_provider; + providers.queries.upstream_monomorphization_for_hash = + upstream_monomorphization_for_hash_provider; providers.queries.wasm_import_module_map = wasm_import_module_map; providers.extern_queries.is_reachable_non_generic = is_reachable_non_generic_provider_extern; providers.extern_queries.upstream_monomorphizations_for = diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index 3a70ee130c27b..83f391a818c4a 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -16,6 +16,7 @@ rustc_errors = { path = "../rustc_errors" } rustc_expand = { path = "../rustc_expand" } rustc_feature = { path = "../rustc_feature" } rustc_fs_util = { path = "../rustc_fs_util" } +rustc_hashes = { path = "../rustc_hashes" } rustc_hir = { path = "../rustc_hir" } rustc_hir_pretty = { path = "../rustc_hir_pretty" } rustc_incremental = { path = "../rustc_incremental" } diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index bd5b3893e4e8a..cf18b1fbcf4fc 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1521,6 +1521,15 @@ impl<'a> CrateMetadataRef<'a> { tcx.arena.alloc_from_iter(self.root.exported_generic_symbols.decode((self, tcx))) } + fn exported_generic_symbol_hashes<'tcx>( + self, + tcx: TyCtxt<'tcx>, + ) -> &'tcx [rustc_hashes::Hash128] { + let table: exported_symbol_hash_map::ExportedSymbolHashTableRef = + self.root.exported_generic_symbol_hashes.decode(&self.cdata.blob); + tcx.arena.alloc_from_iter(table.keys()) + } + fn get_macro(self, tcx: TyCtxt<'_>, id: DefIndex) -> ast::MacroDef { match self.def_kind(tcx, id) { DefKind::Macro(_) => { diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index c6c87534851cd..4825066fefc10 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -407,6 +407,7 @@ provide! { tcx, def_id, other, cdata, stable_order_of_exportable_impls => { tcx.arena.alloc(cdata.get_stable_order_of_exportable_impls(tcx).collect()) } exported_non_generic_symbols => { cdata.exported_non_generic_symbols(tcx) } exported_generic_symbols => { cdata.exported_generic_symbols(tcx) } + exported_generic_symbol_hashes => { cdata.exported_generic_symbol_hashes(tcx) } crate_extern_paths => { cdata.source().paths().cloned().collect() } expn_that_defined => { cdata.get_expn_that_defined(tcx, def_id.index) } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index d8652539d0c51..40500c679f257 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -701,6 +701,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { ) }); + let exported_generic_symbol_hashes = stat!("exported-symbol-hashes", || { + self.encode_exported_generic_symbol_hashes(tcx.exported_generic_symbols(LOCAL_CRATE)) + }); + // Encode the hygiene data. // IMPORTANT: this *must* be the last thing that we encode (other than `SourceMap`). The // process of encoding other items (e.g. `optimized_mir`) may cause us to load data from @@ -768,6 +772,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { stable_order_of_exportable_impls, exported_non_generic_symbols, exported_generic_symbols, + exported_generic_symbol_hashes, interpret_alloc_index, tables, syntax_contexts, @@ -2258,6 +2263,30 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { self.lazy_array(exported_symbols.iter().cloned()) } + fn encode_exported_generic_symbol_hashes( + &mut self, + exported_symbols: &[(ExportedSymbol<'tcx>, SymbolExportInfo)], + ) -> LazyValue { + use exported_symbol_hash_map::{ExportedSymbolHashTable, ExportedSymbolHashTableRef}; + use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; + + let tcx = self.tcx; + let mut table = ExportedSymbolHashTable::with_capacity(exported_symbols.len(), 87); + + for (sym, _) in exported_symbols { + if let Some(instance) = sym.to_instance(tcx) { + let hash = tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + instance.hash_stable(&mut hcx, &mut hasher); + hasher.finish() + }); + table.insert(&hash, &()); + } + } + + self.lazy(ExportedSymbolHashTableRef::Owned(table)) + } + fn encode_dylib_dependency_formats(&mut self) -> LazyArray> { empty_proc_macro!(self); let formats = self.tcx.dependency_formats(()); diff --git a/compiler/rustc_metadata/src/rmeta/exported_symbol_hash_map.rs b/compiler/rustc_metadata/src/rmeta/exported_symbol_hash_map.rs new file mode 100644 index 0000000000000..c067902f2ebb4 --- /dev/null +++ b/compiler/rustc_metadata/src/rmeta/exported_symbol_hash_map.rs @@ -0,0 +1,87 @@ +use rustc_data_structures::owned_slice::OwnedSlice; +use rustc_hashes::Hash128; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; + +use crate::rmeta::EncodeContext; +use crate::rmeta::decoder::BlobDecodeContext; + +#[derive(Clone, Default)] +pub(crate) struct ExportedSymbolHashConfig; + +impl odht::Config for ExportedSymbolHashConfig { + type Key = Hash128; + type Value = (); + + type EncodedKey = [u8; 16]; + type EncodedValue = [u8; 0]; + + type H = odht::UnHashFn; + + #[inline] + fn encode_key(k: &Hash128) -> [u8; 16] { + k.as_u128().to_le_bytes() + } + + #[inline] + fn encode_value(_v: &()) -> [u8; 0] { + [] + } + + #[inline] + fn decode_key(k: &[u8; 16]) -> Hash128 { + Hash128::new(u128::from_le_bytes(*k)) + } + + #[inline] + fn decode_value(_v: &[u8; 0]) {} +} + +pub(crate) type ExportedSymbolHashTable = odht::HashTableOwned; + +pub(crate) enum ExportedSymbolHashTableRef { + /// Zero-copy view into metadata bytes. + OwnedFromMetadata(odht::HashTable), + /// Locally built table, used during encoding. + Owned(ExportedSymbolHashTable), +} + +impl ExportedSymbolHashTableRef { + pub(crate) fn keys(&self) -> Vec { + match self { + Self::OwnedFromMetadata(table) => table.iter().map(|(k, ())| k).collect(), + Self::Owned(table) => table.iter().map(|(k, ())| k).collect(), + } + } +} + +impl<'a, 'tcx> Encodable> for ExportedSymbolHashTableRef { + fn encode(&self, e: &mut EncodeContext<'a, 'tcx>) { + let raw_bytes = match self { + ExportedSymbolHashTableRef::Owned(table) => table.raw_bytes(), + ExportedSymbolHashTableRef::OwnedFromMetadata(_) => { + panic!( + "ExportedSymbolHashTableRef::OwnedFromMetadata variant \ + only exists for deserialization" + ) + } + }; + e.emit_usize(raw_bytes.len()); + e.emit_raw_bytes(raw_bytes); + } +} + +impl<'a> Decodable> for ExportedSymbolHashTableRef { + fn decode(d: &mut BlobDecodeContext<'a>) -> ExportedSymbolHashTableRef { + let len = d.read_usize(); + let pos = d.position(); + let o = d.blob().bytes().clone().slice(|blob| &blob[pos..pos + len]); + + // Advance the decoder's position past the raw bytes we just sliced. + let _ = d.read_raw_bytes(len); + + let inner = odht::HashTable::from_raw_bytes(o).unwrap_or_else(|e| { + panic!("decode error for ExportedSymbolHashTable: {e}"); + }); + ExportedSymbolHashTableRef::OwnedFromMetadata(inner) + } +} diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index af6df0cd6eb61..87ef869dd6c86 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -6,6 +6,7 @@ pub(crate) use decoder::{CrateMetadata, CrateNumMap, MetadataBlob, TargetModifie use def_path_hash_map::DefPathHashMapRef; use encoder::EncodeContext; pub use encoder::{EncodedMetadata, encode_metadata, rendered_const}; +use exported_symbol_hash_map::ExportedSymbolHashTableRef; pub(crate) use parameterized::ParameterizedOverTcx; use rustc_abi::{FieldIdx, ReprOptions, VariantIdx}; use rustc_data_structures::fx::FxHashMap; @@ -49,6 +50,7 @@ use crate::eii::EiiMapEncodedKeyValue; mod decoder; mod def_path_hash_map; mod encoder; +mod exported_symbol_hash_map; mod parameterized; mod table; @@ -276,6 +278,7 @@ pub(crate) struct CrateRoot { stable_order_of_exportable_impls: LazyArray<(DefIndex, usize)>, exported_non_generic_symbols: LazyArray<(ExportedSymbol<'static>, SymbolExportInfo)>, exported_generic_symbols: LazyArray<(ExportedSymbol<'static>, SymbolExportInfo)>, + exported_generic_symbol_hashes: LazyValue, syntax_contexts: SyntaxContextTable, expn_data: ExpnDataTable, diff --git a/compiler/rustc_metadata/src/rmeta/parameterized.rs b/compiler/rustc_metadata/src/rmeta/parameterized.rs index 8a9de07836db4..029c5494783e9 100644 --- a/compiler/rustc_metadata/src/rmeta/parameterized.rs +++ b/compiler/rustc_metadata/src/rmeta/parameterized.rs @@ -73,6 +73,7 @@ trivially_parameterized_over_tcx! { crate::rmeta::RawDefId, crate::rmeta::TraitImpls, crate::rmeta::VariantData, + crate::rmeta::exported_symbol_hash_map::ExportedSymbolHashTableRef, rustc_abi::ReprOptions, rustc_ast::DelimArgs, rustc_hir::Attribute, diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 58fc32078039a..3f4268c39b2d8 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -1,7 +1,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_macros::{Decodable, Encodable, HashStable, TyDecodable, TyEncodable}; -use crate::ty::{self, GenericArgsRef, Ty, TyCtxt}; +use crate::ty::{self, GenericArgsRef, Instance, InstanceKind, Ty, TyCtxt}; /// The SymbolExportLevel of a symbols specifies from which kinds of crates /// the symbol will be exported. `C` symbols will be exported from any @@ -57,6 +57,39 @@ pub enum ExportedSymbol<'tcx> { } impl<'tcx> ExportedSymbol<'tcx> { + /// Convert to the corresponding [`Instance`], if this is a generic symbol. + /// + /// Returns `None` for `NonGeneric`, `ThreadLocalShim`, `NoDefId`, and when + /// required lang items are unavailable. + pub fn to_instance(&self, tcx: TyCtxt<'tcx>) -> Option> { + match *self { + ExportedSymbol::Generic(def_id, args) => { + Some(Instance { def: InstanceKind::Item(def_id), args }) + } + ExportedSymbol::DropGlue(ty) => { + let def_id = tcx.lang_items().drop_in_place_fn()?; + Some(Instance { + def: InstanceKind::DropGlue(def_id, Some(ty)), + args: tcx.mk_args(&[ty.into()]), + }) + } + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + let def_id = tcx.lang_items().async_drop_in_place_fn()?; + Some(Instance { + def: InstanceKind::AsyncDropGlueCtorShim(def_id, ty), + args: tcx.mk_args(&[ty.into()]), + }) + } + ExportedSymbol::AsyncDropGlue(def_id, ty) => Some(Instance { + def: InstanceKind::AsyncDropGlue(def_id, ty), + args: tcx.mk_args(&[ty.into()]), + }), + ExportedSymbol::NonGeneric(..) + | ExportedSymbol::ThreadLocalShim(..) + | ExportedSymbol::NoDefId(..) => None, + } + } + /// This is the symbol name of an instance if it is instantiated in the /// local crate. pub fn symbol_name_for_local_instance(&self, tcx: TyCtxt<'tcx>) -> ty::SymbolName<'tcx> { diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs index de22989144389..78cd6b4285875 100644 --- a/compiler/rustc_middle/src/queries.rs +++ b/compiler/rustc_middle/src/queries.rs @@ -77,6 +77,7 @@ use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::unord::{UnordMap, UnordSet}; use rustc_errors::ErrorGuaranteed; +use rustc_hashes::Hash128; use rustc_hir::attrs::{EiiDecl, EiiImpl, StrippedCfgItem}; use rustc_hir::def::{DefKind, DocLinkResMap}; use rustc_hir::def_id::{ @@ -2011,6 +2012,33 @@ rustc_queries! { desc { "available upstream async-drop-glue for `{:?}`", args } } + /// Per-crate set of hashes of exported generic monomorphizations. + /// Built from a zero-copy odht table in rmeta — no `ExportedSymbol` + /// deserialization is needed to produce this. + query exported_generic_symbol_hashes(cnum: CrateNum) -> &'tcx [Hash128] { + desc { "getting exported generic symbol hashes for crate `{}`", cnum } + separate_provide_extern + } + + /// Hash → candidate crates aggregate, built from all upstream crates' + /// odht hash tables. Only touches fixed-size `Hash128` data — no + /// `ExportedSymbol` deserialization. + query upstream_monomorphization_hashes(_: ()) + -> &'tcx FxIndexMap> + { + arena_cache + desc { "collecting available upstream monomorphization hashes" } + } + + /// Projection query (dep-tracking firewall) for hash-based upstream + /// monomorphization lookup. Projects a single hash from the aggregate + /// built by `upstream_monomorphization_hashes`. + query upstream_monomorphization_for_hash(hash: Hash128) + -> Option<&'tcx smallvec::SmallVec<[CrateNum; 1]>> + { + desc { "looking up upstream monomorphization candidates by hash" } + } + /// Returns a list of all `extern` blocks of a crate. query foreign_modules(_: CrateNum) -> &'tcx FxIndexMap { arena_cache diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs index cccb7d51bd3ed..bf8e7a219e89f 100644 --- a/compiler/rustc_middle/src/query/keys.rs +++ b/compiler/rustc_middle/src/query/keys.rs @@ -3,6 +3,7 @@ use std::ffi::OsStr; use rustc_ast::tokenstream::TokenStream; +use rustc_hashes::Hash128; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId}; use rustc_hir::hir_id::OwnerId; use rustc_query_system::dep_graph::DepNodeIndex; @@ -298,6 +299,12 @@ impl Key for Symbol { } } +impl Key for Hash128 { + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + impl Key for Option { fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { DUMMY_SP diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 0e9dd7dd169cf..a9b4b381928e7 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -2,7 +2,9 @@ use std::fmt; use rustc_data_structures::assert_matches; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_errors::ErrorGuaranteed; +use rustc_hashes::Hash128; use rustc_hir as hir; use rustc_hir::def::{CtorKind, DefKind, Namespace}; use rustc_hir::def_id::{CrateNum, DefId}; @@ -225,15 +227,27 @@ impl<'tcx> Instance<'tcx> { } match self.def { - InstanceKind::Item(def) => tcx - .upstream_monomorphizations_for(def) - .and_then(|monos| monos.get(&self.args).cloned()), - InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args), + InstanceKind::Item(_) + | InstanceKind::DropGlue(_, Some(_)) + | InstanceKind::AsyncDropGlueCtorShim(_, _) => { + let hash: Hash128 = tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + self.hash_stable(&mut hcx, &mut hasher); + hasher.finish() + }); + let candidates = tcx.upstream_monomorphization_for_hash(hash)?; + // Candidates are sorted by StableCrateId. First verified match wins. + for &cnum in candidates.iter() { + for (sym, _) in tcx.exported_generic_symbols(cnum).iter() { + if sym.to_instance(tcx).as_ref() == Some(self) { + return Some(cnum); + } + } + } + None + } InstanceKind::AsyncDropGlue(_, _) => None, InstanceKind::FutureDropPollShim(_, _, _) => None, - InstanceKind::AsyncDropGlueCtorShim(_, _) => { - tcx.upstream_async_drop_glue_for(self.args) - } _ => None, } }