From 7341d68a401199759f4e7ab637a475b02b17a658 Mon Sep 17 00:00:00 2001 From: Diggory Blake Date: Sat, 9 Jul 2016 13:33:03 +0100 Subject: [PATCH 1/2] Produce source package in rust-installer format in addition to vanilla tarball Copy source files from rust code Add missing wildcard Remove unused function Remove use of tar --transform --- mk/dist.mk | 33 ++++++++--- src/bootstrap/Cargo.lock | 56 +++++++++++++++++++ src/bootstrap/Cargo.toml | 1 + src/bootstrap/dist.rs | 116 ++++++++++++++++++++++++++++++++++++++- src/bootstrap/lib.rs | 2 + src/bootstrap/step.rs | 3 + src/bootstrap/util.rs | 29 ++++++++++ 7 files changed, 230 insertions(+), 10 deletions(-) diff --git a/mk/dist.mk b/mk/dist.mk index 9491311ea7c12..e81371037ac2b 100644 --- a/mk/dist.mk +++ b/mk/dist.mk @@ -24,6 +24,7 @@ PKG_NAME := $(CFG_PACKAGE_NAME) STD_PKG_NAME := rust-std-$(CFG_PACKAGE_VERS) DOC_PKG_NAME := rust-docs-$(CFG_PACKAGE_VERS) MINGW_PKG_NAME := rust-mingw-$(CFG_PACKAGE_VERS) +SRC_PKG_NAME := rust-src-$(CFG_PACKAGE_VERS) # License suitable for displaying in a popup LICENSE.txt: $(S)COPYRIGHT $(S)LICENSE-APACHE $(S)LICENSE-MIT @@ -71,10 +72,10 @@ PKG_FILES := \ UNROOTED_PKG_FILES := $(patsubst $(S)%,./%,$(PKG_FILES)) -$(PKG_TAR): $(PKG_FILES) - @$(call E, making dist dir) - $(Q)rm -Rf tmp/dist/$(PKG_NAME) - $(Q)mkdir -p tmp/dist/$(PKG_NAME) +tmp/dist/$$(SRC_PKG_NAME)-image: $(PKG_FILES) + @$(call E, making src image) + $(Q)rm -Rf tmp/dist/$(SRC_PKG_NAME)-image + $(Q)mkdir -p tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src/rust $(Q)tar \ -C $(S) \ -f - \ @@ -87,10 +88,11 @@ $(PKG_TAR): $(PKG_FILES) --exclude=*/llvm/test/*/*/*.ll \ --exclude=*/llvm/test/*/*/*.td \ --exclude=*/llvm/test/*/*/*.s \ - -c $(UNROOTED_PKG_FILES) | tar -x -f - -C tmp/dist/$(PKG_NAME) + -c $(UNROOTED_PKG_FILES) | tar -x -f - -C tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src/rust + +$(PKG_TAR): tmp/dist/$$(SRC_PKG_NAME)-image @$(call E, making $@) - $(Q)tar -czf $(PKG_TAR) -C tmp/dist $(PKG_NAME) - $(Q)rm -Rf tmp/dist/$(PKG_NAME) + $(Q)tar -czf $(PKG_TAR) -C tmp/dist/$(SRC_PKG_NAME)-image/lib/rustlib/src rust --transform 's,^rust,$(PKG_NAME),S' dist-tar-src: $(PKG_TAR) @@ -259,6 +261,19 @@ endef $(foreach host,$(CFG_HOST),\ $(eval $(call DEF_INSTALLER,$(host)))) +dist/$(SRC_PKG_NAME).tar.gz: tmp/dist/$(SRC_PKG_NAME)-image + @$(call E, build: $@) + $(Q)$(S)src/rust-installer/gen-installer.sh \ + --product-name=Rust \ + --rel-manifest-dir=rustlib \ + --success-message=Awesome-Source. \ + --image-dir=tmp/dist/$(SRC_PKG_NAME)-image \ + --work-dir=tmp/dist \ + --output-dir=dist \ + --package-name=$(SRC_PKG_NAME) \ + --component-name=rust-src \ + --legacy-manifest-dirs=rustlib,cargo + # When generating packages for the standard library, we've actually got a lot of # artifacts to choose from. Each of the CFG_HOST compilers will have a copy of # the standard library for each CFG_TARGET, but we only want to generate one @@ -329,8 +344,8 @@ distcheck-docs: dist-docs # Primary targets (dist, distcheck) ###################################################################### -MAYBE_DIST_TAR_SRC=dist-tar-src -MAYBE_DISTCHECK_TAR_SRC=distcheck-tar-src +MAYBE_DIST_TAR_SRC=dist-tar-src dist/$(SRC_PKG_NAME).tar.gz +MAYBE_DISTCHECK_TAR_SRC=distcheck-tar-src dist/$(SRC_PKG_NAME).tar.gz # FIXME #13224: On OS X don't produce tarballs simply because --exclude-vcs don't work. # This is a huge hack because I just don't have time to figure out another solution. diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index 02698d6f7a123..d52577eb228e2 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -11,11 +11,20 @@ dependencies = [ "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.73 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "aho-corasick" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "build_helper" version = "0.1.0" @@ -70,6 +79,14 @@ name = "md5" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "memchr" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num_cpus" version = "0.2.11" @@ -78,11 +95,45 @@ dependencies = [ "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "regex" +version = "0.1.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rustc-serialize" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "thread-id" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "toml" version = "0.1.28" @@ -91,6 +142,11 @@ dependencies = [ "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "utf8-ranges" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi" version = "0.2.6" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 02746034cca69..b19545590b9c0 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -32,3 +32,4 @@ kernel32-sys = "0.2" gcc = { git = "https://github.com/alexcrichton/gcc-rs" } libc = "0.2" md5 = "0.1" +regex = "0.1.73" diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 1cf71c3aaecd6..9d18901eb0002 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -24,7 +24,8 @@ use std::path::{PathBuf, Path}; use std::process::Command; use {Build, Compiler}; -use util::{cp_r, libdir, is_dylib}; +use util::{cp_r, libdir, is_dylib, cp_filtered, copy}; +use regex::{RegexSet, quote}; fn package_vers(build: &Build) -> &str { match &build.config.channel[..] { @@ -284,6 +285,119 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) { t!(fs::remove_dir_all(&image)); } +/// Creates the `rust-src` installer component and the plain source tarball +pub fn rust_src(build: &Build) { + println!("Dist src"); + let plain_name = format!("rustc-{}-src", package_vers(build)); + let name = format!("rust-src-{}", package_vers(build)); + let image = tmpdir(build).join(format!("{}-image", name)); + let _ = fs::remove_dir_all(&image); + + let dst = image.join("lib/rustlib/src"); + let dst_src = dst.join("rust"); + let plain_dst_src = dst.join(&plain_name); + t!(fs::create_dir_all(&dst_src)); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "Makefile.in" + ]; + let src_dirs = [ + "man", + "src", + "mk" + ]; + + // Exclude paths matching these wildcard expressions + let excludes = [ + // exclude-vcs + "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules", ".gitattributes", ".cvsignore", + ".svn", ".arch-ids", "{arch}", "=RELEASE-ID", "=meta-update", "=update", ".bzr", + ".bzrignore", ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs", + // extensions + "*~", "*.pyc", + // misc + "llvm/test/*/*.ll", + "llvm/test/*/*.td", + "llvm/test/*/*.s", + "llvm/test/*/*/*.ll", + "llvm/test/*/*/*.td", + "llvm/test/*/*/*.s" + ]; + + // Construct a set of regexes for efficiently testing whether paths match one of the above + // expressions. + let regex_set = t!(RegexSet::new( + // This converts a wildcard expression to a regex + excludes.iter().map(|&s| { + // Prefix ensures that matching starts on a path separator boundary + r"^(.*[\\/])?".to_owned() + ( + // Escape the expression to produce a regex matching exactly that string + "e(s) + // Replace slashes with a pattern matching either forward or backslash + .replace(r"/", r"[\\/]") + // Replace wildcards with a pattern matching a single path segment, ie. containing + // no slashes. + .replace(r"\*", r"[^\\/]*") + // Suffix anchors to the end of the path + ) + "$" + }) + )); + + // Create a filter which skips files which match the regex set or contain invalid unicode + let filter_fn = move |path: &Path| { + if let Some(path) = path.to_str() { + !regex_set.is_match(path) + } else { + false + } + }; + + // Copy the directories using our filter + for item in &src_dirs { + let dst = &dst_src.join(item); + t!(fs::create_dir(dst)); + cp_filtered(&build.src.join(item), dst, &filter_fn); + } + // Copy the files normally + for item in &src_files { + copy(&build.src.join(item), &dst_src.join(item)); + } + + // Create source tarball in rust-installer format + let mut cmd = Command::new("sh"); + cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh"))) + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=Awesome-Source.") + .arg(format!("--image-dir={}", sanitize_sh(&image))) + .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build)))) + .arg(format!("--output-dir={}", sanitize_sh(&distdir(build)))) + .arg(format!("--package-name={}", name)) + .arg("--component-name=rust-src") + .arg("--legacy-manifest-dirs=rustlib,cargo"); + build.run(&mut cmd); + + // Rename directory, so that root folder of tarball has the correct name + t!(fs::rename(&dst_src, &plain_dst_src)); + + // Create plain source tarball + let mut cmd = Command::new("tar"); + cmd.arg("-czf").arg(sanitize_sh(&distdir(build).join(&format!("{}.tar.gz", plain_name)))) + .arg(&plain_name) + .current_dir(&dst); + build.run(&mut cmd); + + t!(fs::remove_dir_all(&image)); +} + fn install(src: &Path, dstdir: &Path, perms: u32) { let dst = dstdir.join(src.file_name().unwrap()); t!(fs::create_dir_all(dstdir)); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 5d61abe5e086a..3ef8cb74dfa6d 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -26,6 +26,7 @@ extern crate md5; extern crate num_cpus; extern crate rustc_serialize; extern crate toml; +extern crate regex; use std::cell::RefCell; use std::collections::HashMap; @@ -451,6 +452,7 @@ impl Build { DistMingw { _dummy } => dist::mingw(self, target.target), DistRustc { stage } => dist::rustc(self, stage, target.target), DistStd { compiler } => dist::std(self, &compiler, target.target), + DistSrc { _dummy } => dist::rust_src(self), DebuggerScripts { stage } => { let compiler = Compiler::new(stage, target.target); diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index f715ceb16d71b..8d3cb36166b70 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -140,6 +140,7 @@ macro_rules! targets { (dist_mingw, DistMingw { _dummy: () }), (dist_rustc, DistRustc { stage: u32 }), (dist_std, DistStd { compiler: Compiler<'a> }), + (dist_src, DistSrc { _dummy: () }), // Misc targets (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }), @@ -568,12 +569,14 @@ impl<'a> Step<'a> { vec![self.libtest(compiler)] } } + Source::DistSrc { _dummy: _ } => Vec::new(), Source::Dist { stage } => { let mut base = Vec::new(); for host in build.config.host.iter() { let host = self.target(host); + base.push(host.dist_src(())); base.push(host.dist_rustc(stage)); if host.target.contains("windows-gnu") { base.push(host.dist_mingw(())); diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index b5230132bcb66..dfc1c7a243b5b 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -67,6 +67,35 @@ pub fn cp_r(src: &Path, dst: &Path) { } } +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. Unwanted files or directories can be skipped +/// by returning `false` from the filter function. +pub fn cp_filtered bool>(src: &Path, dst: &Path, filter: &F) { + // Inner function does the actual work + fn recurse bool>(src: &Path, dst: &Path, relative: &Path, filter: &F) { + for f in t!(fs::read_dir(src)) { + let f = t!(f); + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + let relative = relative.join(name); + // Only copy file or directory if the filter function returns true + if filter(&relative) { + if t!(f.file_type()).is_dir() { + let _ = fs::remove_dir_all(&dst); + t!(fs::create_dir(&dst)); + recurse(&path, &dst, &relative, filter); + } else { + let _ = fs::remove_file(&dst); + copy(&path, &dst); + } + } + } + } + // Immediately recurse with an empty relative path + recurse(src, dst, Path::new(""), filter) +} + /// Given an executable called `name`, return the filename for the /// executable for a particular target. pub fn exe(name: &str, target: &str) -> String { From b3908d08ee477f550dcb0e3d23305dee77e2258b Mon Sep 17 00:00:00 2001 From: Diggory Blake Date: Sat, 13 Aug 2016 22:36:04 +0100 Subject: [PATCH 2/2] Fix make-tidy lock file checks --- src/tools/tidy/src/cargo_lock.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tools/tidy/src/cargo_lock.rs b/src/tools/tidy/src/cargo_lock.rs index 4324db489b7f8..165dd52758ecc 100644 --- a/src/tools/tidy/src/cargo_lock.rs +++ b/src/tools/tidy/src/cargo_lock.rs @@ -9,6 +9,7 @@ // except according to those terms. use std::path::Path; +use std::ffi::OsStr; const CARGO_LOCK: &'static str = "Cargo.lock"; @@ -18,14 +19,15 @@ pub fn check(path: &Path, bad: &mut bool) { super::walk(path, &mut |path| super::filter_dirs(path) || path.ends_with("src/test"), &mut |file| { - let name = file.file_name().unwrap().to_string_lossy(); - if name == CARGO_LOCK { + if let Some(CARGO_LOCK) = file.file_name().and_then(OsStr::to_str) { let rel_path = file.strip_prefix(path).unwrap(); + let git_friendly_path = rel_path.to_str().unwrap().replace("\\", "/"); let ret_code = Command::new("git") - .arg("diff-index") - .arg("--quiet") + .arg("diff") + .arg("--exit-code") + .arg("--patch") .arg("HEAD") - .arg(rel_path) + .arg(&git_friendly_path) .current_dir(path) .status() .unwrap_or_else(|e| {