Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove source distribution filename from cache #8907

Merged
merged 1 commit into from
Nov 8, 2024
Merged

Remove source distribution filename from cache #8907

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 18 additions & 50 deletions crates/uv-distribution/src/source/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use uv_configuration::{BuildKind, BuildOutput, SourceStrategy};
use uv_distribution_filename::{SourceDistExtension, WheelFilename};
use uv_distribution_types::{
BuildableSource, DirectorySourceUrl, FileLocation, GitSourceUrl, HashPolicy, Hashed,
PathSourceUrl, RemoteSource, SourceDist, SourceUrl,
PathSourceUrl, SourceDist, SourceUrl,
};
use uv_extract::hash::Hasher;
use uv_fs::{rename_with_retry, write_atomic, LockedFile};
Expand Down Expand Up @@ -58,6 +58,9 @@ pub(crate) const LOCAL_REVISION: &str = "revision.rev";
/// The name of the file that contains the cached distribution metadata, encoded via `MsgPack`.
pub(crate) const METADATA: &str = "metadata.msgpack";

/// The directory within each entry under which to store the unpacked source distribution.
pub(crate) const SOURCE: &str = "src";

impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
/// Initialize a [`SourceDistributionBuilder`] from a [`BuildContext`].
pub(crate) fn new(build_context: &'a T) -> Self {
Expand Down Expand Up @@ -125,7 +128,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&dist.file.filename,
&url,
&cache_shard,
None,
Expand All @@ -138,8 +140,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -148,7 +148,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&filename,
&dist.url,
&cache_shard,
dist.subdirectory.as_deref(),
Expand Down Expand Up @@ -186,11 +185,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -199,7 +193,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&filename,
resource.url,
&cache_shard,
resource.subdirectory,
Expand Down Expand Up @@ -281,22 +274,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await;
}

self.url_metadata(
source,
&dist.file.filename,
&url,
&cache_shard,
None,
dist.ext,
hashes,
client,
)
.boxed_local()
.await?
self.url_metadata(source, &url, &cache_shard, None, dist.ext, hashes, client)
.boxed_local()
.await?
}
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -305,7 +287,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url_metadata(
source,
&filename,
&dist.url,
&cache_shard,
dist.subdirectory.as_deref(),
Expand Down Expand Up @@ -336,11 +317,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -349,7 +325,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url_metadata(
source,
&filename,
resource.url,
&cache_shard,
resource.subdirectory,
Expand Down Expand Up @@ -403,7 +378,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url<'data>(
&self,
source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url,
cache_shard: &CacheShard,
subdirectory: Option<&'data Path>,
Expand All @@ -416,7 +390,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Fetch the revision for the source distribution.
let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client)
.url_revision(source, ext, url, cache_shard, hashes, client)
.await?;

// Before running the build, check that the hashes match.
Expand All @@ -431,7 +405,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename);
let source_dist_entry = cache_shard.entry(SOURCE);

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
Expand All @@ -452,7 +426,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else {
self.heal_url_revision(
source,
filename,
ext,
url,
&source_dist_entry,
Expand Down Expand Up @@ -507,7 +480,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_metadata<'data>(
&self,
source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url,
cache_shard: &CacheShard,
subdirectory: Option<&'data Path>,
Expand All @@ -519,7 +491,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Fetch the revision for the source distribution.
let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client)
.url_revision(source, ext, url, cache_shard, hashes, client)
.await?;

// Before running the build, check that the hashes match.
Expand All @@ -534,7 +506,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename);
let source_dist_entry = cache_shard.entry(SOURCE);

// If the metadata is static, return it.
if let Some(metadata) =
Expand Down Expand Up @@ -562,7 +534,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else {
self.heal_url_revision(
source,
filename,
ext,
url,
&source_dist_entry,
Expand Down Expand Up @@ -644,7 +615,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_revision(
&self,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
url: &Url,
cache_shard: &CacheShard,
Expand All @@ -670,9 +640,9 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Download the source distribution.
debug!("Downloading source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry(filename);
let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes)
.download_archive(response, source, ext, entry.path(), hashes)
.await?;

Ok(revision.with_hashes(hashes))
Expand Down Expand Up @@ -743,7 +713,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source");
let source_entry = cache_shard.entry(SOURCE);

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
Expand Down Expand Up @@ -832,7 +802,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source");
let source_entry = cache_shard.entry(SOURCE);

// If the metadata is static, return it.
if let Some(metadata) =
Expand Down Expand Up @@ -957,7 +927,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Unzip the archive to a temporary directory.
debug!("Unpacking source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry("source");
let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self
.persist_archive(&resource.path, resource.ext, entry.path(), hashes)
.await?;
Expand Down Expand Up @@ -1568,7 +1538,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn heal_url_revision(
&self,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
url: &Url,
entry: &CacheEntry,
Expand All @@ -1581,7 +1550,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let download = |response| {
async {
let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes)
.download_archive(response, source, ext, entry.path(), hashes)
.await?;
if hashes != revision.hashes() {
return Err(Error::CacheHeal(source.to_string()));
Expand Down Expand Up @@ -1610,7 +1579,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self,
response: Response,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
target: &Path,
hashes: HashPolicy<'_>,
Expand All @@ -1632,7 +1600,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);

// Download and unzip the source distribution into a temporary directory.
let span = info_span!("download_source_dist", filename = filename, source_dist = %source);
let span = info_span!("download_source_dist", source_dist = %source);
uv_extract::stream::archive(&mut hasher, ext, temp_dir.path()).await?;
drop(span);

Expand Down
Loading