From 798732f26292b3e701ca926351ce4622aa4b4641 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Tue, 2 May 2017 18:08:17 -0700 Subject: [PATCH 01/21] Rewrite gen-installer.sh in Rust --- .gitignore | 5 +- Cargo.toml | 8 ++ gen-installer.sh | 322 +---------------------------------------------- src/generator.rs | 211 +++++++++++++++++++++++++++++++ src/lib.rs | 21 ++++ src/main.rs | 58 +++++++++ src/main.yml | 62 +++++++++ 7 files changed, 365 insertions(+), 322 deletions(-) create mode 100644 Cargo.toml create mode 100644 src/generator.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/main.yml diff --git a/.gitignore b/.gitignore index 31ed630..fb017f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *~ -tmp \ No newline at end of file +tmp +target/ +**/*.rs.bk +Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..ef1656f --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[package] +authors = ["The Rust Project Developers"] +name = "installer" +version = "0.0.0" + +[dependencies.clap] +version = "2.22.1" +features = ["yaml"] diff --git a/gen-installer.sh b/gen-installer.sh index a85f1aa..60fac3b 100755 --- a/gen-installer.sh +++ b/gen-installer.sh @@ -11,193 +11,6 @@ set -ue -msg() { - echo "gen-installer: ${1-}" -} - -step_msg() { - msg - msg "$1" - msg -} - -warn() { - echo "gen-installer: WARNING: $1" >&2 -} - -err() { - echo "gen-installer: error: $1" >&2 - exit 1 -} - -need_ok() { - if [ $? -ne 0 ] - then - err "$1" - fi -} - -need_cmd() { - if command -v $1 >/dev/null 2>&1 - then msg "found $1" - else err "need $1" - fi -} - -putvar() { - local t - local tlen - eval t=\$$1 - eval tlen=\${#$1} - if [ $tlen -gt 35 ] - then - printf "gen-installer: %-20s := %.35s ...\n" $1 "$t" - else - printf "gen-installer: %-20s := %s %s\n" $1 "$t" - fi -} - -valopt() { - VAL_OPTIONS="$VAL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - if [ $HELP -eq 0 ] - then - local uop=$(echo $op | tr '[:lower:]' '[:upper:]' | tr '\-' '\_') - local v="CFG_${uop}" - eval $v="$default" - for arg in $CFG_ARGS - do - if echo "$arg" | grep -q -- "--$op=" - then - local val=$(echo "$arg" | cut -f2 -d=) - eval $v=$val - fi - done - putvar $v - else - if [ -z "$default" ] - then - default="" - fi - op="${default}=[${default}]" - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -opt() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - local flag="" - - if [ $default -eq 0 ] - then - flag="enable" - else - flag="disable" - doc="don't $doc" - fi - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${flag}-${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - flag=$(echo $flag | tr 'a-z' 'A-Z') - local v="CFG_${flag}_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$flag-$op" "$doc" - fi -} - -flag() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - shift - local doc="$*" - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - local v="CFG_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -validate_opt () { - for arg in $CFG_ARGS - do - local is_arg_valid=0 - for option in $BOOL_OPTIONS - do - if test --disable-$option = $arg - then - is_arg_valid=1 - fi - if test --enable-$option = $arg - then - is_arg_valid=1 - fi - if test --$option = $arg - then - is_arg_valid=1 - fi - done - for option in $VAL_OPTIONS - do - if echo "$arg" | grep -q -- "--$option=" - then - is_arg_valid=1 - fi - done - if [ "$arg" = "--help" ] - then - echo - echo "No more help available for Configure options," - echo "check the Wiki or join our IRC channel" - break - else - if test $is_arg_valid -eq 0 - then - err "Option '$arg' is not recognized" - fi - fi - done -} - # Prints the absolute path of a directory to stdout abs_path() { local path="$1" @@ -207,138 +20,5 @@ abs_path() { (unset CDPATH && cd "$path" > /dev/null && pwd) } -msg "looking for programs" -msg - -need_cmd cp -need_cmd rm -need_cmd mkdir -need_cmd echo -need_cmd tr -need_cmd awk - -CFG_ARGS="$@" - -HELP=0 -if [ "$1" = "--help" ] -then - HELP=1 - shift - echo - echo "Usage: $0 [options]" - echo - echo "Options:" - echo -else - step_msg "processing arguments" -fi - -OPTIONS="" -BOOL_OPTIONS="" -VAL_OPTIONS="" - -valopt product-name "Product" "The name of the product, for display" -valopt component-name "component" "The name of the component, distinct from other installed components" -valopt package-name "package" "The name of the package, tarball" -valopt rel-manifest-dir "${CFG_PACKAGE_NAME}lib" "The directory under lib/ where the manifest lives" -valopt success-message "Installed." "The string to print after successful installation" -valopt legacy-manifest-dirs "" "Places to look for legacy manifests to uninstall" -valopt non-installed-overlay "" "Directory containing files that should not be installed" -valopt bulk-dirs "" "Path prefixes of directories that should be installed/uninstalled in bulk" -valopt image-dir "./install-image" "The directory containing the installation medium" -valopt work-dir "./workdir" "The directory to do temporary work" -valopt output-dir "./dist" "The location to put the final image and tarball" - -if [ $HELP -eq 1 ] -then - echo - exit 0 -fi - -step_msg "validating arguments" -validate_opt - src_dir="$(abs_path $(dirname "$0"))" - -rust_installer_version=`cat "$src_dir/rust-installer-version"` - -if [ ! -d "$CFG_IMAGE_DIR" ] -then - err "image dir $CFG_IMAGE_DIR does not exist" -fi - -mkdir -p "$CFG_WORK_DIR" -need_ok "couldn't create work dir" - -rm -Rf "$CFG_WORK_DIR/$CFG_PACKAGE_NAME" -need_ok "couldn't delete work package dir" - -mkdir -p "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$CFG_COMPONENT_NAME" -need_ok "couldn't create work package dir" - -cp -r "$CFG_IMAGE_DIR/"* "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$CFG_COMPONENT_NAME" -need_ok "couldn't copy source image" - -# Create the manifest -manifest=`(cd "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$CFG_COMPONENT_NAME" && find . -type f | sed 's/^\.\///') | sort` - -# Remove files in bulk dirs -bulk_dirs=`echo "$CFG_BULK_DIRS" | tr "," " "` -for bulk_dir in $bulk_dirs; do - bulk_dir=`echo "$bulk_dir" | sed s/\\\//\\\\\\\\\\\//g` - manifest=`echo "$manifest" | sed /^$bulk_dir/d` -done - -# Add 'file:' installation directives, skipping empty lines. -manifest=`echo "$manifest" | sed /^$/d | sed s/^/file:/` - -# Add 'dir:' directives -for bulk_dir in $bulk_dirs; do - manifest=`echo "$manifest" && echo "dir:$bulk_dir"` -done - -# The above step may have left a leading empty line if there were only -# bulk dirs. Remove it. -manifest=`echo "$manifest" | sed /^$/d` - -manifest_file="$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$CFG_COMPONENT_NAME/manifest.in" -component_file="$CFG_WORK_DIR/$CFG_PACKAGE_NAME/components" -version_file="$CFG_WORK_DIR/$CFG_PACKAGE_NAME/rust-installer-version" - -# Write the manifest -echo "$manifest" > "$manifest_file" - -# Write the component name -echo "$CFG_COMPONENT_NAME" > "$component_file" - -# Write the installer version (only used by combine-installers.sh) -echo "$rust_installer_version" > "$version_file" - -# Copy the overlay -if [ -n "$CFG_NON_INSTALLED_OVERLAY" ]; then - overlay_files=`(cd "$CFG_NON_INSTALLED_OVERLAY" && find . -type f)` - for f in $overlay_files; do - if [ -e "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$f" ]; then err "overlay $f exists"; fi - - cp "$CFG_NON_INSTALLED_OVERLAY/$f" "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$f" - need_ok "failed to copy overlay $f" - done -fi - -# Generate the install script -"$src_dir/gen-install-script.sh" \ - --product-name="$CFG_PRODUCT_NAME" \ - --rel-manifest-dir="$CFG_REL_MANIFEST_DIR" \ - --success-message="$CFG_SUCCESS_MESSAGE" \ - --legacy-manifest-dirs="$CFG_LEGACY_MANIFEST_DIRS" \ - --output-script="$CFG_WORK_DIR/$CFG_PACKAGE_NAME/install.sh" - -need_ok "failed to generate install script" - -mkdir -p "$CFG_OUTPUT_DIR" -need_ok "couldn't create output dir" - -"$src_dir/make-tarballs.sh" \ - --work-dir="$CFG_WORK_DIR" \ - --input="$CFG_PACKAGE_NAME" \ - --output="$CFG_OUTPUT_DIR/$CFG_PACKAGE_NAME" +cargo run --manifest-path="$src_dir/Cargo.toml" -- generate "$@" diff --git a/src/generator.rs b/src/generator.rs new file mode 100644 index 0000000..ff2cf3a --- /dev/null +++ b/src/generator.rs @@ -0,0 +1,211 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[derive(Debug)] +pub struct Generator { + product_name: String, + component_name: String, + package_name: String, + rel_manifest_dir: String, + success_message: String, + legacy_manifest_dirs: String, + non_installed_overlay: String, + bulk_dirs: String, + image_dir: String, + work_dir: String, + output_dir: String, +} + +impl Default for Generator { + fn default() -> Generator { + Generator { + product_name: "Product".into(), + component_name: "component".into(), + package_name: "package".into(), + rel_manifest_dir: "packagelib".into(), + success_message: "Installed.".into(), + legacy_manifest_dirs: "".into(), + non_installed_overlay: "".into(), + bulk_dirs: "".into(), + image_dir: "./install_image".into(), + work_dir: "./workdir".into(), + output_dir: "./dist".into(), + } + } +} + +impl Generator { + /// The name of the product, for display + pub fn product_name(&mut self, value: String) -> &mut Self { + self.product_name = value; + self + } + + /// The name of the component, distinct from other installed components + pub fn component_name(&mut self, value: String) -> &mut Self { + self.component_name = value; + self + } + + /// The name of the package, tarball + pub fn package_name(&mut self, value: String) -> &mut Self { + self.package_name = value; + self + } + + /// The directory under lib/ where the manifest lives + pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { + self.rel_manifest_dir = value; + self + } + + /// The string to print after successful installation + pub fn success_message(&mut self, value: String) -> &mut Self { + self.success_message = value; + self + } + + /// Places to look for legacy manifests to uninstall + pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { + self.legacy_manifest_dirs = value; + self + } + + /// Directory containing files that should not be installed + pub fn non_installed_overlay(&mut self, value: String) -> &mut Self { + self.non_installed_overlay = value; + self + } + + /// Path prefixes of directories that should be installed/uninstalled in bulk + pub fn bulk_dirs(&mut self, value: String) -> &mut Self { + self.bulk_dirs = value; + self + } + + /// The directory containing the installation medium + pub fn image_dir(&mut self, value: String) -> &mut Self { + self.image_dir = value; + self + } + + /// The directory to do temporary work + pub fn work_dir(&mut self, value: String) -> &mut Self { + self.work_dir = value; + self + } + + /// The location to put the final image and tarball + pub fn output_dir(&mut self, value: String) -> &mut Self { + self.output_dir = value; + self + } + + /// Generate the actual installer tarball + pub fn run(self) -> io::Result<()> { + let src_dir = Path::new(::SOURCE_DIRECTORY); + fs::read_dir(&src_dir)?; + + fs::create_dir_all(&self.work_dir)?; + + let package_dir = Path::new(&self.work_dir).join(&self.package_name); + if package_dir.exists() { + fs::remove_dir_all(&package_dir)?; + } + + let component_dir = package_dir.join(&self.component_name); + fs::create_dir_all(&component_dir)?; + let mut files = cp_r(self.image_dir.as_ref(), &component_dir)?; + + // Filter out files that are covered by bulk dirs. + let bulk_dirs: Vec<_> = self.bulk_dirs.split(',').filter(|s| !s.is_empty()).collect(); + files.retain(|f| !bulk_dirs.iter().any(|d| f.starts_with(d))); + + // Write the manifest + let manifest = fs::File::create(component_dir.join("manifest.in"))?; + for file in files { + writeln!(&manifest, "file:{}", file.display())?; + } + for dir in bulk_dirs { + writeln!(&manifest, "dir:{}", dir)?; + } + drop(manifest); + + // Write the component name + let components = fs::File::create(package_dir.join("components"))?; + writeln!(&components, "{}", self.component_name)?; + drop(components); + + // Write the installer version (only used by combine-installers.sh) + let version = fs::File::create(package_dir.join("rust-installer-version"))?; + writeln!(&version, "{}", ::RUST_INSTALLER_VERSION)?; + drop(version); + + // Copy the overlay + if !self.non_installed_overlay.is_empty() { + cp_r(self.non_installed_overlay.as_ref(), &package_dir)?; + } + + // Generate the install script (TODO: run this in-process!) + let output_script = package_dir.join("install.sh"); + let status = Command::new(src_dir.join("gen-install-script.sh")) + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--rel-manifest-dir={}", self.rel_manifest_dir)) + .arg(format!("--success-message={}", self.success_message)) + .arg(format!("--legacy-manifest-dirs={}", self.legacy_manifest_dirs)) + .arg(format!("--output-script={}", output_script.display())) + .status()?; + if !status.success() { + let msg = format!("failed to generate install script: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + // Make the tarballs (TODO: run this in-process!) + fs::create_dir_all(&self.output_dir)?; + let output = Path::new(&self.output_dir).join(&self.package_name); + let status = Command::new(src_dir.join("make-tarballs.sh")) + .arg(format!("--work-dir={}", self.work_dir)) + .arg(format!("--input={}", self.package_name)) + .arg(format!("--output={}", output.display())) + .status()?; + if !status.success() { + let msg = format!("failed to make tarballs: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + Ok(()) + } +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. Returns a list of files written relative to `dst`. +pub fn cp_r(src: &Path, dst: &Path) -> io::Result> { + let mut files = vec![]; + for f in fs::read_dir(src)? { + let f = f?; + let path = f.path(); + let name = PathBuf::from(f.file_name()); + let dst = dst.join(&name); + if f.file_type()?.is_dir() { + fs::create_dir(&dst)?; + let subfiles = cp_r(&path, &dst)?; + files.extend(subfiles.into_iter().map(|f| name.join(f))); + } else { + fs::copy(&path, &dst)?; + files.push(name); + } + } + Ok(files) +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c81dc12 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,21 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +mod generator; + +pub use generator::Generator; + +/// The base source directory +/// (FIXME: statically compiling this means we can't ship installer binaries!) +const SOURCE_DIRECTORY: &'static str = env!("CARGO_MANIFEST_DIR"); + +/// The installer version, output only to be used by combine-installers.sh. +/// (should match `SOURCE_DIRECTORY/rust_installer_version`) +pub const RUST_INSTALLER_VERSION: u32 = 3; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..bdc2dd5 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,58 @@ +#[macro_use] +extern crate clap; +extern crate installer; + +use clap::{App, ArgMatches}; +use installer::Generator; + +fn main() { + let yaml = load_yaml!("main.yml"); + let matches = App::from_yaml(yaml).get_matches(); + + match matches.subcommand() { + ("generate", Some(matches)) => generate(matches), + _ => unreachable!(), + } +} + +fn generate(matches: &ArgMatches) { + let mut gen = Generator::default(); + matches + .value_of("product-name") + .map(|s| gen.product_name(s.into())); + matches + .value_of("component-name") + .map(|s| gen.component_name(s.into())); + matches + .value_of("package-name") + .map(|s| gen.package_name(s.into())); + matches + .value_of("rel-manifest-dir") + .map(|s| gen.rel_manifest_dir(s.into())); + matches + .value_of("success-message") + .map(|s| gen.success_message(s.into())); + matches + .value_of("legacy-manifest-dirs") + .map(|s| gen.legacy_manifest_dirs(s.into())); + matches + .value_of("non-installed-overlay") + .map(|s| gen.non_installed_overlay(s.into())); + matches + .value_of("bulk-dirs") + .map(|s| gen.bulk_dirs(s.into())); + matches + .value_of("image-dir") + .map(|s| gen.image_dir(s.into())); + matches + .value_of("work-dir") + .map(|s| gen.work_dir(s.into())); + matches + .value_of("output-dir") + .map(|s| gen.output_dir(s.into())); + + if let Err(e) = gen.run() { + println!("failed to generate installer: {}", e); + std::process::exit(1); + } +} diff --git a/src/main.yml b/src/main.yml new file mode 100644 index 0000000..ead98b6 --- /dev/null +++ b/src/main.yml @@ -0,0 +1,62 @@ +name: installer +settings: + - ArgRequiredElseHelp +subcommands: + - generate: + args: + - product-name: + help: The name of the product, for display + long: product-name + takes_value: true + value_name: NAME + - component-name: + help: The name of the component, distinct from other installed components + long: component-name + takes_value: true + value_name: NAME + - package-name: + help: The name of the package, tarball + long: package-name + takes_value: true + value_name: NAME + - rel-manifest-dir: + help: The directory under lib/ where the manifest lives + long: rel-manifest-dir + takes_value: true + value_name: DIR + - success-message: + help: The string to print after successful installation + long: success-message + takes_value: true + value_name: MESSAGE + - legacy-manifest-dirs: + help: Places to look for legacy manifests to uninstall + long: legacy-manifest-dirs + takes_value: true + value_name: DIRS + - non-installed-overlay: + help: Directory containing files that should not be installed + long: non-installed-overlay + takes_value: true + value_name: DIR + - bulk-dirs: + help: Path prefixes of directories that should be installed/uninstalled in bulk + long: bulk-dirs + takes_value: true + value_name: DIRS + - image-dir: + help: The directory containing the installation medium + long: image-dir + takes_value: true + value_name: DIR + - work-dir: + help: The directory to do temporary work + long: work-dir + takes_value: true + value_name: DIR + - output-dir: + help: The location to put the final image and tarball + long: output-dir + takes_value: true + value_name: DIR + From 1ea5467d932d2898d2e06140991bc1728d528ae3 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 3 May 2017 13:30:22 -0700 Subject: [PATCH 02/21] Rewrite gen-install-script.sh in Rust --- gen-install-script.sh | 261 +----------------------------------------- src/generator.rs | 21 ++-- src/lib.rs | 2 + src/main.rs | 27 ++++- src/main.yml | 29 +++++ src/scripter.rs | 104 +++++++++++++++++ 6 files changed, 172 insertions(+), 272 deletions(-) create mode 100644 src/scripter.rs diff --git a/gen-install-script.sh b/gen-install-script.sh index 620fcf7..1420814 100755 --- a/gen-install-script.sh +++ b/gen-install-script.sh @@ -9,200 +9,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -set -u - -if [ -x /bin/echo ]; then - ECHO='/bin/echo' -else - ECHO='echo' -fi - -msg() { - echo "gen-install-script: ${1-}" -} - -step_msg() { - msg - msg "$1" - msg -} - -warn() { - echo "gen-install-script: WARNING: $1" >&2 -} - -err() { - echo "gen-install-script: error: $1" >&2 - exit 1 -} - -need_ok() { - if [ $? -ne 0 ] - then - err "$1" - fi -} - -need_cmd() { - if command -v $1 >/dev/null 2>&1 - then msg "found $1" - else err "need $1" - fi -} - -putvar() { - local t - local tlen - eval t=\$$1 - eval tlen=\${#$1} - if [ $tlen -gt 35 ] - then - printf "gen-install-script: %-20s := %.35s ...\n" $1 "$t" - else - printf "gen-install-script: %-20s := %s %s\n" $1 "$t" - fi -} - -valopt() { - VAL_OPTIONS="$VAL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - if [ $HELP -eq 0 ] - then - local uop=$(echo $op | tr '[:lower:]' '[:upper:]' | tr '\-' '\_') - local v="CFG_${uop}" - eval $v="$default" - for arg in $CFG_ARGS - do - if echo "$arg" | grep -q -- "--$op=" - then - local val=$(echo "$arg" | cut -f2 -d=) - eval $v=$val - fi - done - putvar $v - else - if [ -z "$default" ] - then - default="" - fi - op="${default}=[${default}]" - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -opt() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - local flag="" - - if [ $default -eq 0 ] - then - flag="enable" - else - flag="disable" - doc="don't $doc" - fi - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${flag}-${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - flag=$(echo $flag | tr 'a-z' 'A-Z') - local v="CFG_${flag}_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$flag-$op" "$doc" - fi -} - -flag() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - shift - local doc="$*" - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - local v="CFG_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -validate_opt () { - for arg in $CFG_ARGS - do - local is_arg_valid=0 - for option in $BOOL_OPTIONS - do - if test --disable-$option = $arg - then - is_arg_valid=1 - fi - if test --enable-$option = $arg - then - is_arg_valid=1 - fi - if test --$option = $arg - then - is_arg_valid=1 - fi - done - for option in $VAL_OPTIONS - do - if echo "$arg" | grep -q -- "--$option=" - then - is_arg_valid=1 - fi - done - if [ "$arg" = "--help" ] - then - echo - echo "No more help available for Configure options," - echo "check the Wiki or join our IRC channel" - break - else - if test $is_arg_valid -eq 0 - then - err "Option '$arg' is not recognized" - fi - fi - done -} +set -ue # Prints the absolute path of a directory to stdout abs_path() { @@ -213,69 +20,5 @@ abs_path() { (unset CDPATH && cd "$path" > /dev/null && pwd) } -msg "looking for install programs" -msg - -need_cmd sed -need_cmd chmod -need_cmd cat - -CFG_ARGS="$@" - -HELP=0 -if [ "$1" = "--help" ] -then - HELP=1 - shift - echo - echo "Usage: $0 [options]" - echo - echo "Options:" - echo -else - step_msg "processing arguments" -fi - -OPTIONS="" -BOOL_OPTIONS="" -VAL_OPTIONS="" - -valopt product-name "Product" "The name of the product, for display" -valopt rel-manifest-dir "manifestlib" "The directory under lib/ where the manifest lives" -valopt success-message "Installed." "The string to print after successful installation" -valopt output-script "install.sh" "The name of the output script" -valopt legacy-manifest-dirs "" "Places to look for legacy manifests to uninstall" - -if [ $HELP -eq 1 ] -then - echo - exit 0 -fi - -step_msg "validating arguments" -validate_opt - src_dir="$(abs_path $(dirname "$0"))" - -rust_installer_version=`cat "$src_dir/rust-installer-version"` - -# Replace dashes in the success message with spaces (our arg handling botches spaces) -product_name=`echo "$CFG_PRODUCT_NAME" | sed "s/-/ /g"` - -# Replace dashes in the success message with spaces (our arg handling botches spaces) -success_message=`echo "$CFG_SUCCESS_MESSAGE" | sed "s/-/ /g"` - -script_template=`cat "$src_dir/install-template.sh"` - -# Using /bin/echo because under sh emulation dash *seems* to escape \n, which screws up the template -script=`$ECHO "$script_template"` -script=`$ECHO "$script" | sed "s/%%TEMPLATE_PRODUCT_NAME%%/\"$product_name\"/"` -script=`$ECHO "$script" | sed "s/%%TEMPLATE_REL_MANIFEST_DIR%%/$CFG_REL_MANIFEST_DIR/"` -script=`$ECHO "$script" | sed "s/%%TEMPLATE_SUCCESS_MESSAGE%%/\"$success_message\"/"` -script=`$ECHO "$script" | sed "s/%%TEMPLATE_LEGACY_MANIFEST_DIRS%%/\"$CFG_LEGACY_MANIFEST_DIRS\"/"` -script=`$ECHO "$script" | sed "s/%%TEMPLATE_RUST_INSTALLER_VERSION%%/\"$rust_installer_version\"/"` - -$ECHO "$script" > "$CFG_OUTPUT_SCRIPT" -need_ok "couldn't write script" -chmod u+x "$CFG_OUTPUT_SCRIPT" -need_ok "couldn't chmod script" +cargo run --manifest-path="$src_dir/Cargo.toml" -- script "$@" diff --git a/src/generator.rs b/src/generator.rs index ff2cf3a..2a01119 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -12,6 +12,7 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::Command; +use super::Scripter; #[derive(Debug)] pub struct Generator { @@ -158,19 +159,15 @@ impl Generator { cp_r(self.non_installed_overlay.as_ref(), &package_dir)?; } - // Generate the install script (TODO: run this in-process!) + // Generate the install script let output_script = package_dir.join("install.sh"); - let status = Command::new(src_dir.join("gen-install-script.sh")) - .arg(format!("--product-name={}", self.product_name)) - .arg(format!("--rel-manifest-dir={}", self.rel_manifest_dir)) - .arg(format!("--success-message={}", self.success_message)) - .arg(format!("--legacy-manifest-dirs={}", self.legacy_manifest_dirs)) - .arg(format!("--output-script={}", output_script.display())) - .status()?; - if !status.success() { - let msg = format!("failed to generate install script: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + let mut scripter = Scripter::default(); + scripter.product_name(self.product_name) + .rel_manifest_dir(self.rel_manifest_dir) + .success_message(self.success_message) + .legacy_manifest_dirs(self.legacy_manifest_dirs) + .output_script(output_script.to_str().unwrap().into()); + scripter.run()?; // Make the tarballs (TODO: run this in-process!) fs::create_dir_all(&self.output_dir)?; diff --git a/src/lib.rs b/src/lib.rs index c81dc12..89d0182 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,10 @@ // except according to those terms. mod generator; +mod scripter; pub use generator::Generator; +pub use scripter::Scripter; /// The base source directory /// (FIXME: statically compiling this means we can't ship installer binaries!) diff --git a/src/main.rs b/src/main.rs index bdc2dd5..66c23b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ extern crate clap; extern crate installer; use clap::{App, ArgMatches}; -use installer::Generator; +use installer::*; fn main() { let yaml = load_yaml!("main.yml"); @@ -11,6 +11,7 @@ fn main() { match matches.subcommand() { ("generate", Some(matches)) => generate(matches), + ("script", Some(matches)) => script(matches), _ => unreachable!(), } } @@ -56,3 +57,27 @@ fn generate(matches: &ArgMatches) { std::process::exit(1); } } + +fn script(matches: &ArgMatches) { + let mut scr = Scripter::default(); + matches + .value_of("product-name") + .map(|s| scr.product_name(s.into())); + matches + .value_of("rel-manifest-dir") + .map(|s| scr.rel_manifest_dir(s.into())); + matches + .value_of("success-message") + .map(|s| scr.success_message(s.into())); + matches + .value_of("legacy-manifest-dirs") + .map(|s| scr.legacy_manifest_dirs(s.into())); + matches + .value_of("output-script") + .map(|s| scr.output_script(s.into())); + + if let Err(e) = scr.run() { + println!("failed to generate installation script: {}", e); + std::process::exit(1); + } +} diff --git a/src/main.yml b/src/main.yml index ead98b6..937322a 100644 --- a/src/main.yml +++ b/src/main.yml @@ -3,6 +3,7 @@ settings: - ArgRequiredElseHelp subcommands: - generate: + about: Generate a complete installer tarball args: - product-name: help: The name of the product, for display @@ -59,4 +60,32 @@ subcommands: long: output-dir takes_value: true value_name: DIR + - script: + about: Generate an installation script + args: + - product-name: + help: The name of the product, for display + long: product-name + takes_value: true + value_name: NAME + - rel-manifest-dir: + help: The directory under lib/ where the manifest lives + long: rel-manifest-dir + takes_value: true + value_name: DIR + - success-message: + help: The string to print after successful installation + long: success-message + takes_value: true + value_name: MESSAGE + - legacy-manifest-dirs: + help: Places to look for legacy manifests to uninstall + long: legacy-manifest-dirs + takes_value: true + value_name: DIRS + - output-script: + help: The name of the output script + long: output-script + takes_value: true + value_name: FILE diff --git a/src/scripter.rs b/src/scripter.rs new file mode 100644 index 0000000..a5b51ba --- /dev/null +++ b/src/scripter.rs @@ -0,0 +1,104 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::fs; +use std::io::{self, Write}; + +// Needed to set the script mode to executable. +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +// FIXME: what about Windows? Are default ACLs executable? + +const TEMPLATE: &'static str = include_str!("../install-template.sh"); + +#[derive(Debug)] +pub struct Scripter { + product_name: String, + rel_manifest_dir: String, + success_message: String, + legacy_manifest_dirs: String, + output_script: String, +} + +impl Default for Scripter { + fn default() -> Scripter { + Scripter { + product_name: "Product".into(), + rel_manifest_dir: "manifestlib".into(), + success_message: "Installed.".into(), + legacy_manifest_dirs: "".into(), + output_script: "install.sh".into(), + } + } +} + +impl Scripter { + /// The name of the product, for display + pub fn product_name(&mut self, value: String) -> &mut Self { + self.product_name = value; + self + } + + /// The directory under lib/ where the manifest lives + pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { + self.rel_manifest_dir = value; + self + } + + /// The string to print after successful installation + pub fn success_message(&mut self, value: String) -> &mut Self { + self.success_message = value; + self + } + + /// Places to look for legacy manifests to uninstall + pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { + self.legacy_manifest_dirs = value; + self + } + + /// The name of the output script + pub fn output_script(&mut self, value: String) -> &mut Self { + self.output_script = value; + self + } + + /// Generate the actual installer script + pub fn run(self) -> io::Result<()> { + // Replace dashes in the success message with spaces (our arg handling botches spaces) + // (TODO: still needed? kept for compatibility for now...) + let product_name = self.product_name.replace('-', " "); + + // Replace dashes in the success message with spaces (our arg handling botches spaces) + // (TODO: still needed? kept for compatibility for now...) + let success_message = self.success_message.replace('-', " "); + + let script = TEMPLATE + .replace("%%TEMPLATE_PRODUCT_NAME%%", &sh_quote(&product_name)) + .replace("%%TEMPLATE_REL_MANIFEST_DIR%%", &self.rel_manifest_dir) + .replace("%%TEMPLATE_SUCCESS_MESSAGE%%", &sh_quote(&success_message)) + .replace("%%TEMPLATE_LEGACY_MANIFEST_DIRS%%", &sh_quote(&self.legacy_manifest_dirs)) + .replace("%%TEMPLATE_RUST_INSTALLER_VERSION%%", &sh_quote(&::RUST_INSTALLER_VERSION)); + + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + if cfg!(unix) { + options.mode(0o755); + } + let output = options.open(self.output_script)?; + writeln!(&output, "{}", script) + } +} + +fn sh_quote(s: &T) -> String { + // We'll single-quote the whole thing, so first replace single-quotes with + // '"'"' (leave quoting, double-quote one `'`, re-enter single-quoting) + format!("'{}'", s.to_string().replace('\'', r#"'"'"'"#)) +} From ab8c80f1e4b86364e7154a8319d2637dd9a01c50 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 3 May 2017 15:28:54 -0700 Subject: [PATCH 03/21] Use walkdir for recursive traversal --- Cargo.toml | 5 +++- src/generator.rs | 78 +++++++++++++++++++++++++++++------------------- src/lib.rs | 2 ++ 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ef1656f..a1521e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ authors = ["The Rust Project Developers"] name = "installer" version = "0.0.0" +[dependencies] +walkdir = "1.0.7" + [dependencies.clap] -version = "2.22.1" features = ["yaml"] +version = "2.22.1" diff --git a/src/generator.rs b/src/generator.rs index 2a01119..303ac6e 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -10,8 +10,10 @@ use std::fs; use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; +use walkdir::WalkDir; + use super::Scripter; #[derive(Debug)] @@ -126,23 +128,10 @@ impl Generator { fs::remove_dir_all(&package_dir)?; } + // Copy the image and write the manifest let component_dir = package_dir.join(&self.component_name); fs::create_dir_all(&component_dir)?; - let mut files = cp_r(self.image_dir.as_ref(), &component_dir)?; - - // Filter out files that are covered by bulk dirs. - let bulk_dirs: Vec<_> = self.bulk_dirs.split(',').filter(|s| !s.is_empty()).collect(); - files.retain(|f| !bulk_dirs.iter().any(|d| f.starts_with(d))); - - // Write the manifest - let manifest = fs::File::create(component_dir.join("manifest.in"))?; - for file in files { - writeln!(&manifest, "file:{}", file.display())?; - } - for dir in bulk_dirs { - writeln!(&manifest, "dir:{}", dir)?; - } - drop(manifest); + copy_and_manifest(self.image_dir.as_ref(), &component_dir, &self.bulk_dirs)?; // Write the component name let components = fs::File::create(package_dir.join("components"))?; @@ -156,7 +145,7 @@ impl Generator { // Copy the overlay if !self.non_installed_overlay.is_empty() { - cp_r(self.non_installed_overlay.as_ref(), &package_dir)?; + copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?; } // Generate the install script @@ -187,22 +176,49 @@ impl Generator { } /// Copies the `src` directory recursively to `dst`. Both are assumed to exist -/// when this function is called. Returns a list of files written relative to `dst`. -pub fn cp_r(src: &Path, dst: &Path) -> io::Result> { - let mut files = vec![]; - for f in fs::read_dir(src)? { - let f = f?; - let path = f.path(); - let name = PathBuf::from(f.file_name()); - let dst = dst.join(&name); - if f.file_type()?.is_dir() { +/// when this function is called. +fn copy_recursive(src: &Path, dst: &Path) -> io::Result<()> { + copy_with_callback(src, dst, |_, _| Ok(())) +} + +/// Copies the `src` directory recursively to `dst`, writing `manifest.in` too. +fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> io::Result<()> { + let manifest = fs::File::create(dst.join("manifest.in"))?; + let bulk_dirs: Vec<_> = bulk_dirs.split(',') + .filter(|s| !s.is_empty()) + .map(Path::new).collect(); + + copy_with_callback(src, dst, |path, file_type| { + if file_type.is_dir() { + if bulk_dirs.contains(&path) { + writeln!(&manifest, "dir:{}", path.display())?; + } + } else { + if !bulk_dirs.iter().any(|d| path.starts_with(d)) { + writeln!(&manifest, "file:{}", path.display())?; + } + } + Ok(()) + }) +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. Invokes a callback for each path visited. +fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> io::Result<()> + where F: FnMut(&Path, fs::FileType) -> io::Result<()> +{ + for entry in WalkDir::new(src).min_depth(1) { + let entry = entry?; + let file_type = entry.file_type(); + let path = entry.path().strip_prefix(src).unwrap(); + let dst = dst.join(path); + + if file_type.is_dir() { fs::create_dir(&dst)?; - let subfiles = cp_r(&path, &dst)?; - files.extend(subfiles.into_iter().map(|f| name.join(f))); } else { - fs::copy(&path, &dst)?; - files.push(name); + fs::copy(entry.path(), dst)?; } + callback(&path, file_type)?; } - Ok(files) + Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 89d0182..2a0d099 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +extern crate walkdir; + mod generator; mod scripter; From e6fedf91ebe47e9e572d6353e57cc692cff1a695 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 3 May 2017 16:38:07 -0700 Subject: [PATCH 04/21] Rewrite make-tarballs.sh in Rust --- make-tarballs.sh | 262 +---------------------------------------------- src/generator.rs | 21 ++-- src/lib.rs | 6 +- src/main.rs | 19 ++++ src/main.yml | 18 ++++ src/tarballer.rs | 201 ++++++++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 278 deletions(-) create mode 100644 src/tarballer.rs diff --git a/make-tarballs.sh b/make-tarballs.sh index b206a27..e9f88cc 100755 --- a/make-tarballs.sh +++ b/make-tarballs.sh @@ -11,193 +11,6 @@ set -ue -msg() { - echo "make-tarballs: ${1-}" -} - -step_msg() { - msg - msg "$1" - msg -} - -warn() { - echo "make-tarballs: WARNING: $1" >&2 -} - -err() { - echo "make-tarballs: error: $1" >&2 - exit 1 -} - -need_ok() { - if [ $? -ne 0 ] - then - err "$1" - fi -} - -need_cmd() { - if command -v $1 >/dev/null 2>&1 - then msg "found $1" - else err "need $1" - fi -} - -putvar() { - local t - local tlen - eval t=\$$1 - eval tlen=\${#$1} - if [ $tlen -gt 35 ] - then - printf "make-tarballs: %-20s := %.35s ...\n" $1 "$t" - else - printf "make-tarballs: %-20s := %s %s\n" $1 "$t" - fi -} - -valopt() { - VAL_OPTIONS="$VAL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - if [ $HELP -eq 0 ] - then - local uop=$(echo $op | tr '[:lower:]' '[:upper:]' | tr '\-' '\_') - local v="CFG_${uop}" - eval $v="$default" - for arg in $CFG_ARGS - do - if echo "$arg" | grep -q -- "--$op=" - then - local val=$(echo "$arg" | cut -f2 -d=) - eval $v=$val - fi - done - putvar $v - else - if [ -z "$default" ] - then - default="" - fi - op="${op}=[${default}]" - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -opt() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - local flag="" - - if [ $default -eq 0 ] - then - flag="enable" - else - flag="disable" - doc="don't $doc" - fi - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${flag}-${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - flag=$(echo $flag | tr 'a-z' 'A-Z') - local v="CFG_${flag}_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$flag-$op" "$doc" - fi -} - -flag() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - shift - local doc="$*" - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - local v="CFG_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -validate_opt () { - for arg in $CFG_ARGS - do - local is_arg_valid=0 - for option in $BOOL_OPTIONS - do - if test --disable-$option = $arg - then - is_arg_valid=1 - fi - if test --enable-$option = $arg - then - is_arg_valid=1 - fi - if test --$option = $arg - then - is_arg_valid=1 - fi - done - for option in $VAL_OPTIONS - do - if echo "$arg" | grep -q -- "--$option=" - then - is_arg_valid=1 - fi - done - if [ "$arg" = "--help" ] - then - echo - echo "No more help available for Configure options," - echo "check the Wiki or join our IRC channel" - break - else - if test $is_arg_valid -eq 0 - then - err "Option '$arg' is not recognized" - fi - fi - done -} - # Prints the absolute path of a directory to stdout abs_path() { local path="$1" @@ -207,76 +20,5 @@ abs_path() { (unset CDPATH && cd "$path" > /dev/null && pwd) } -msg "looking for programs" -msg - -need_cmd tar -need_cmd rm -need_cmd mkdir -need_cmd echo -need_cmd tr -need_cmd find -need_cmd rev -need_cmd sort -need_cmd gzip - -# need_cmd xz || need_cmd 7z -if command -v xz >/dev/null 2>&1 -then msg "found xz" -else need_cmd 7z -fi - -CFG_ARGS="$@" - -HELP=0 -if [ "$1" = "--help" ] -then - HELP=1 - shift - echo - echo "Usage: $0 [options]" - echo - echo "Options:" - echo -else - step_msg "processing arguments" -fi - -OPTIONS="" -BOOL_OPTIONS="" -VAL_OPTIONS="" - -valopt input "package" "The input folder to be compressed" -valopt output "./dist" "The prefix of the tarballs" -valopt work-dir "./workdir" "The folder in which the input is to be found" - -if [ $HELP -eq 1 ] -then - echo - exit 0 -fi - -step_msg "validating arguments" -validate_opt - -rm -Rf "$CFG_OUTPUT.tar.gz" -need_ok "couldn't delete old gz tarball" - -rm -Rf "$CFG_OUTPUT.tar.xz" -need_ok "couldn't delete old xz tarball" - -# Make a tarball -cd "$CFG_WORK_DIR" - -tar -cf "$CFG_OUTPUT.tar" "$CFG_INPUT" - -need_ok "failed to tar" - -if command -v xz >/dev/null 2>&1 -then xz -9 --keep "$CFG_OUTPUT.tar" -else 7z a -bd -txz -mx=9 -mmt=off "$CFG_OUTPUT.tar.xz" "$CFG_OUTPUT.tar" -fi -need_ok "failed to xz" - -gzip "$CFG_OUTPUT.tar" -need_ok "failed to gzip" +src_dir="$(abs_path $(dirname "$0"))" +cargo run --manifest-path="$src_dir/Cargo.toml" -- tarball "$@" diff --git a/src/generator.rs b/src/generator.rs index 303ac6e..c3accc7 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -11,10 +11,10 @@ use std::fs; use std::io::{self, Write}; use std::path::Path; -use std::process::Command; use walkdir::WalkDir; use super::Scripter; +use super::Tarballer; #[derive(Debug)] pub struct Generator { @@ -118,9 +118,6 @@ impl Generator { /// Generate the actual installer tarball pub fn run(self) -> io::Result<()> { - let src_dir = Path::new(::SOURCE_DIRECTORY); - fs::read_dir(&src_dir)?; - fs::create_dir_all(&self.work_dir)?; let package_dir = Path::new(&self.work_dir).join(&self.package_name); @@ -158,18 +155,14 @@ impl Generator { .output_script(output_script.to_str().unwrap().into()); scripter.run()?; - // Make the tarballs (TODO: run this in-process!) + // Make the tarballs fs::create_dir_all(&self.output_dir)?; let output = Path::new(&self.output_dir).join(&self.package_name); - let status = Command::new(src_dir.join("make-tarballs.sh")) - .arg(format!("--work-dir={}", self.work_dir)) - .arg(format!("--input={}", self.package_name)) - .arg(format!("--output={}", output.display())) - .status()?; - if !status.success() { - let msg = format!("failed to make tarballs: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + let mut tarballer = Tarballer::default(); + tarballer.work_dir(self.work_dir) + .input(self.package_name) + .output(output.to_str().unwrap().into()); + tarballer.run()?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 2a0d099..277fd03 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,13 +12,11 @@ extern crate walkdir; mod generator; mod scripter; +mod tarballer; pub use generator::Generator; pub use scripter::Scripter; - -/// The base source directory -/// (FIXME: statically compiling this means we can't ship installer binaries!) -const SOURCE_DIRECTORY: &'static str = env!("CARGO_MANIFEST_DIR"); +pub use tarballer::Tarballer; /// The installer version, output only to be used by combine-installers.sh. /// (should match `SOURCE_DIRECTORY/rust_installer_version`) diff --git a/src/main.rs b/src/main.rs index 66c23b8..575e8e6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ fn main() { match matches.subcommand() { ("generate", Some(matches)) => generate(matches), ("script", Some(matches)) => script(matches), + ("tarball", Some(matches)) => tarball(matches), _ => unreachable!(), } } @@ -81,3 +82,21 @@ fn script(matches: &ArgMatches) { std::process::exit(1); } } + +fn tarball(matches: &ArgMatches) { + let mut tar = Tarballer::default(); + matches + .value_of("input") + .map(|s| tar.input(s.into())); + matches + .value_of("output") + .map(|s| tar.output(s.into())); + matches + .value_of("work-dir") + .map(|s| tar.work_dir(s.into())); + + if let Err(e) = tar.run() { + println!("failed to generate tarballs: {}", e); + std::process::exit(1); + } +} diff --git a/src/main.yml b/src/main.yml index 937322a..c6edebb 100644 --- a/src/main.yml +++ b/src/main.yml @@ -88,4 +88,22 @@ subcommands: long: output-script takes_value: true value_name: FILE + - tarball: + about: Generate package tarballs + args: + - input: + help: The input folder to be compressed + long: input + takes_value: true + value_name: NAME + - output: + help: The prefix of the tarballs + long: output + takes_value: true + value_name: PATH + - work-dir: + help: The folder in which the input is to be found + long: work-dir + takes_value: true + value_name: DIR diff --git a/src/tarballer.rs b/src/tarballer.rs new file mode 100644 index 0000000..f642796 --- /dev/null +++ b/src/tarballer.rs @@ -0,0 +1,201 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::env; +use std::ffi::{OsString, OsStr}; +use std::fs; +use std::io::{self, Write}; +use std::path::Path; +use std::process::{Command, Stdio}; +use walkdir::WalkDir; + +#[derive(Debug)] +pub struct Tarballer { + input: String, + output: String, + work_dir: String, +} + +impl Default for Tarballer { + fn default() -> Tarballer { + Tarballer { + input: "package".into(), + output: "./dist".into(), + work_dir: "./workdir.".into(), + } + } +} + +impl Tarballer { + /// The input folder to be compressed + pub fn input(&mut self, value: String) -> &mut Self { + self.input = value; + self + } + + /// The prefix of the tarballs + pub fn output(&mut self, value: String) -> &mut Self { + self.output = value; + self + } + + /// The fold in which the input is to be found + pub fn work_dir(&mut self, value: String) -> &mut Self { + self.work_dir = value; + self + } + + /// Generate the actual tarballs + pub fn run(self) -> io::Result<()> { + let path = get_path()?; + need_cmd(&path, "tar")?; + need_cmd(&path, "gzip")?; + let have_xz = need_either_cmd(&path, "xz", "7z")?; + + let tar = self.output + ".tar"; + let tar_gz = tar.clone() + ".gz"; + let tar_xz = tar.clone() + ".xz"; + + // Remove any existing files + for file in &[&tar, &tar_gz, &tar_xz] { + if Path::new(file).exists() { + fs::remove_file(file)?; + } + } + + // Sort files by their suffix, to group files with the same name from + // different locations (likely identical) and files with the same + // extension (likely containing similar data). + let mut paths = get_recursive_paths(self.work_dir.as_ref(), self.input.as_ref())?; + paths.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); + + // Write the tar file + let mut child = Command::new("tar") + .arg("-cf") + .arg(&tar) + .arg("-T") + .arg("-") + .stdin(Stdio::piped()) + .current_dir(&self.work_dir) + .spawn()?; + if let Some(stdin) = child.stdin.as_mut() { + for path in paths { + writeln!(stdin, "{}", path)?; + } + } + let status = child.wait()?; + if !status.success() { + let msg = format!("failed to make tarball: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + // Write the .tar.xz file + let status = if have_xz { + Command::new("xz") + .arg("-9") + .arg("--keep") + .arg(&tar) + .status()? + } else { + Command::new("7z") + .arg("a") + .arg("-bd") + .arg("-txz") + .arg("-mx=9") + .arg("-mmt=off") + .arg(&tar_xz) + .arg(&tar) + .status()? + }; + if !status.success() { + let msg = format!("failed to make tar.xz: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + // Write the .tar.gz file (removing the .tar) + let status = Command::new("gzip") + .arg(&tar) + .status()?; + if !status.success() { + let msg = format!("failed to make tar.gz: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + Ok(()) + } +} + +fn get_path() -> io::Result { + let path = env::var_os("PATH").unwrap_or(OsString::new()); + // On Windows, quotes are invalid characters for filename paths, and if + // one is present as part of the PATH then that can lead to the system + // being unable to identify the files properly. See + // https://github.com/rust-lang/rust/issues/34959 for more details. + if cfg!(windows) { + if path.to_string_lossy().contains("\"") { + let msg = "PATH contains invalid character '\"'"; + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + } + Ok(path) +} + +fn have_cmd(path: &OsStr, cmd: &str) -> bool { + for path in env::split_paths(path) { + let target = path.join(cmd); + let cmd_alt = cmd.to_string() + ".exe"; + if target.is_file() || + target.with_extension("exe").exists() || + target.join(cmd_alt).exists() { + return true; + } + } + false +} + +fn need_cmd(path: &OsStr, cmd: &str) -> io::Result<()> { + if have_cmd(path, cmd) { + Ok(()) + } else { + let msg = format!("couldn't find required command: '{}'", cmd); + Err(io::Error::new(io::ErrorKind::NotFound, msg)) + } +} + +fn need_either_cmd(path: &OsStr, cmd1: &str, cmd2: &str) -> io::Result { + if have_cmd(path, cmd1) { + Ok(true) + } else if have_cmd(path, cmd2) { + Ok(false) + } else { + let msg = format!("couldn't find either command: '{}' or '{}'", cmd1, cmd2); + Err(io::Error::new(io::ErrorKind::NotFound, msg)) + } +} + +fn get_recursive_paths(root: &Path, name: &Path) -> io::Result> { + let mut paths = vec![]; + for entry in WalkDir::new(root.join(name)).min_depth(1) { + let entry = entry?; + let path = entry.path().strip_prefix(root).unwrap(); + let path = path.to_str().unwrap().to_owned(); + + if entry.file_type().is_dir() { + // Include only empty dirs, as others get add via their contents. + // FIXME: do we really need empty dirs at all? + if fs::read_dir(entry.path())?.next().is_none() { + paths.push(path); + } + } else { + paths.push(path); + } + } + Ok(paths) +} From 1aad328d8924b0cc4a711b41f665831aef639e0c Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 14:53:37 -0700 Subject: [PATCH 05/21] Rewrite combine-installers.sh in Rust --- combine-installers.sh | 314 +----------------------------------------- src/combiner.rs | 192 ++++++++++++++++++++++++++ src/generator.rs | 29 +--- src/lib.rs | 3 + src/main.rs | 37 +++++ src/main.yml | 48 +++++++ src/tarballer.rs | 52 +------ src/util.rs | 91 ++++++++++++ 8 files changed, 376 insertions(+), 390 deletions(-) create mode 100644 src/combiner.rs create mode 100644 src/util.rs diff --git a/combine-installers.sh b/combine-installers.sh index 76d10a7..e56dc8d 100755 --- a/combine-installers.sh +++ b/combine-installers.sh @@ -9,194 +9,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -set -u - -msg() { - echo "combine-installers: ${1-}" -} - -step_msg() { - msg - msg "$1" - msg -} - -warn() { - echo "combine-installers: WARNING: $1" >&2 -} - -err() { - echo "combine-installers: error: $1" >&2 - exit 1 -} - -need_ok() { - if [ $? -ne 0 ] - then - err "$1" - fi -} - -need_cmd() { - if command -v $1 >/dev/null 2>&1 - then msg "found $1" - else err "need $1" - fi -} - -putvar() { - local t - local tlen - eval t=\$$1 - eval tlen=\${#$1} - if [ $tlen -gt 35 ] - then - printf "combine-installers: %-20s := %.35s ...\n" $1 "$t" - else - printf "combine-installers: %-20s := %s %s\n" $1 "$t" - fi -} - -valopt() { - VAL_OPTIONS="$VAL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - if [ $HELP -eq 0 ] - then - local uop=$(echo $op | tr '[:lower:]' '[:upper:]' | tr '\-' '\_') - local v="CFG_${uop}" - eval $v="$default" - for arg in $CFG_ARGS - do - if echo "$arg" | grep -q -- "--$op=" - then - local val=$(echo "$arg" | cut -f2 -d=) - eval $v=$val - fi - done - putvar $v - else - if [ -z "$default" ] - then - default="" - fi - op="${default}=[${default}]" - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -opt() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - local default=$2 - shift - shift - local doc="$*" - local flag="" - - if [ $default -eq 0 ] - then - flag="enable" - else - flag="disable" - doc="don't $doc" - fi - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${flag}-${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - flag=$(echo $flag | tr 'a-z' 'A-Z') - local v="CFG_${flag}_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$flag-$op" "$doc" - fi -} - -flag() { - BOOL_OPTIONS="$BOOL_OPTIONS $1" - - local op=$1 - shift - local doc="$*" - - if [ $HELP -eq 0 ] - then - for arg in $CFG_ARGS - do - if [ "$arg" = "--${op}" ] - then - op=$(echo $op | tr 'a-z-' 'A-Z_') - local v="CFG_${op}" - eval $v=1 - putvar $v - fi - done - else - if [ ! -z "$META" ] - then - op="$op=<$META>" - fi - printf " --%-30s %s\n" "$op" "$doc" - fi -} - -validate_opt () { - for arg in $CFG_ARGS - do - local is_arg_valid=0 - for option in $BOOL_OPTIONS - do - if test --disable-$option = $arg - then - is_arg_valid=1 - fi - if test --enable-$option = $arg - then - is_arg_valid=1 - fi - if test --$option = $arg - then - is_arg_valid=1 - fi - done - for option in $VAL_OPTIONS - do - if echo "$arg" | grep -q -- "--$option=" - then - is_arg_valid=1 - fi - done - if [ "$arg" = "--help" ] - then - echo - echo "No more help available for Configure options," - echo "check the Wiki or join our IRC channel" - break - else - if test $is_arg_valid -eq 0 - then - err "Option '$arg' is not recognized" - fi - fi - done -} +set -ue # Prints the absolute path of a directory to stdout abs_path() { @@ -207,128 +20,5 @@ abs_path() { (unset CDPATH && cd "$path" > /dev/null && pwd) } -msg "looking for programs" -msg - -need_cmd tar -need_cmd cp -need_cmd rm -need_cmd mkdir -need_cmd echo -need_cmd tr - -CFG_ARGS="$@" - -HELP=0 -if [ "$1" = "--help" ] -then - HELP=1 - shift - echo - echo "Usage: $0 [options]" - echo - echo "Options:" - echo -else - step_msg "processing arguments" -fi - -OPTIONS="" -BOOL_OPTIONS="" -VAL_OPTIONS="" - -valopt product-name "Product" "The name of the product, for display" -valopt package-name "package" "The name of the package, tarball" -valopt rel-manifest-dir "${CFG_PACKAGE_NAME}lib" "The directory under lib/ where the manifest lives" -valopt success-message "Installed." "The string to print after successful installation" -valopt legacy-manifest-dirs "" "Places to look for legacy manifests to uninstall" -valopt input-tarballs "" "Installers to combine" -valopt non-installed-overlay "" "Directory containing files that should not be installed" -valopt work-dir "./workdir" "The directory to do temporary work and put the final image" -valopt output-dir "./dist" "The location to put the final tarball" - -if [ $HELP -eq 1 ] -then - echo - exit 0 -fi - -step_msg "validating arguments" -validate_opt - src_dir="$(abs_path $(dirname "$0"))" - -rust_installer_version=`cat "$src_dir/rust-installer-version"` - -# Create the work directory for the new installer -mkdir -p "$CFG_WORK_DIR" -need_ok "couldn't create work dir" - -rm -Rf "$CFG_WORK_DIR/$CFG_PACKAGE_NAME" -need_ok "couldn't delete work package dir" - -mkdir -p "$CFG_WORK_DIR/$CFG_PACKAGE_NAME" -need_ok "couldn't create work package dir" - -input_tarballs=`echo "$CFG_INPUT_TARBALLS" | sed 's/,/ /g'` - -# Merge each installer into the work directory of the new installer -for input_tarball in $input_tarballs; do - - # Extract the input tarballs - tar xzf $input_tarball -C "$CFG_WORK_DIR" - need_ok "failed to extract tarball" - - # Verify the version number - pkg_name=`echo "$input_tarball" | sed s/\.tar\.gz//g` - pkg_name=`basename $pkg_name` - version=`cat "$CFG_WORK_DIR/$pkg_name/rust-installer-version"` - if [ "$rust_installer_version" != "$version" ]; then - err "incorrect installer version in $input_tarball" - fi - - # Copy components to new combined installer - components=`cat "$CFG_WORK_DIR/$pkg_name/components"` - for component in $components; do - - # All we need to do is copy the component directory - cp -R "$CFG_WORK_DIR/$pkg_name/$component" "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$component" - need_ok "failed to copy component $component" - - # Merge the component name - echo "$component" >> "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/components" - need_ok "failed to merge component $component" - done -done - -# Write the version number -echo "$rust_installer_version" > "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/rust-installer-version" - -# Copy the overlay -if [ -n "$CFG_NON_INSTALLED_OVERLAY" ]; then - overlay_files=`(cd "$CFG_NON_INSTALLED_OVERLAY" && find . -type f)` - for f in $overlay_files; do - if [ -e "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$f" ]; then err "overlay $f exists"; fi - - cp "$CFG_NON_INSTALLED_OVERLAY/$f" "$CFG_WORK_DIR/$CFG_PACKAGE_NAME/$f" - need_ok "failed to copy overlay $f" - done -fi - -# Generate the install script -"$src_dir/gen-install-script.sh" \ - --product-name="$CFG_PRODUCT_NAME" \ - --rel-manifest-dir="$CFG_REL_MANIFEST_DIR" \ - --success-message="$CFG_SUCCESS_MESSAGE" \ - --legacy-manifest-dirs="$CFG_LEGACY_MANIFEST_DIRS" \ - --output-script="$CFG_WORK_DIR/$CFG_PACKAGE_NAME/install.sh" - -need_ok "failed to generate install script" - -mkdir -p "$CFG_OUTPUT_DIR" -need_ok "couldn't create output dir" - -"$src_dir/make-tarballs.sh" \ - --work-dir="$CFG_WORK_DIR" \ - --input="$CFG_PACKAGE_NAME" \ - --output="$CFG_OUTPUT_DIR/$CFG_PACKAGE_NAME" +cargo run --manifest-path="$src_dir/Cargo.toml" -- combine "$@" diff --git a/src/combiner.rs b/src/combiner.rs new file mode 100644 index 0000000..20e0ec2 --- /dev/null +++ b/src/combiner.rs @@ -0,0 +1,192 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::fs; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::process::Command; + +use super::Scripter; +use super::Tarballer; +use util::*; + +#[derive(Debug)] +pub struct Combiner { + product_name: String, + package_name: String, + rel_manifest_dir: String, + success_message: String, + legacy_manifest_dirs: String, + input_tarballs: String, + non_installed_overlay: String, + work_dir: String, + output_dir: String, +} + +impl Default for Combiner { + fn default() -> Combiner { + Combiner { + product_name: "Product".into(), + package_name: "package".into(), + rel_manifest_dir: "packagelib".into(), + success_message: "Installed.".into(), + legacy_manifest_dirs: "".into(), + input_tarballs: "".into(), + non_installed_overlay: "".into(), + work_dir: "./workdir".into(), + output_dir: "./dist".into(), + } + } +} + +impl Combiner { + /// The name of the product, for display + pub fn product_name(&mut self, value: String) -> &mut Self { + self.product_name = value; + self + } + + /// The name of the package, tarball + pub fn package_name(&mut self, value: String) -> &mut Self { + self.package_name = value; + self + } + + /// The directory under lib/ where the manifest lives + pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { + self.rel_manifest_dir = value; + self + } + + /// The string to print after successful installation + pub fn success_message(&mut self, value: String) -> &mut Self { + self.success_message = value; + self + } + + /// Places to look for legacy manifests to uninstall + pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { + self.legacy_manifest_dirs = value; + self + } + + /// Installers to combine + pub fn input_tarballs(&mut self, value: String) -> &mut Self { + self.input_tarballs = value; + self + } + + /// Directory containing files that should not be installed + pub fn non_installed_overlay(&mut self, value: String) -> &mut Self { + self.non_installed_overlay = value; + self + } + + /// The directory to do temporary work + pub fn work_dir(&mut self, value: String) -> &mut Self { + self.work_dir = value; + self + } + + /// The location to put the final image and tarball + pub fn output_dir(&mut self, value: String) -> &mut Self { + self.output_dir = value; + self + } + + /// Generate the actual installer tarball + pub fn run(self) -> io::Result<()> { + let path = get_path()?; + need_cmd(&path, "tar")?; + + fs::create_dir_all(&self.work_dir)?; + + let package_dir = Path::new(&self.work_dir).join(&self.package_name); + if package_dir.exists() { + fs::remove_dir_all(&package_dir)?; + } + fs::create_dir_all(&package_dir)?; + + // Merge each installer into the work directory of the new installer + let components = fs::File::create(package_dir.join("components"))?; + for input_tarball in self.input_tarballs.split(',').map(str::trim).filter(|s| !s.is_empty()) { + // Extract the input tarballs + let status = Command::new("tar") + .arg("xzf") + .arg(&input_tarball) + .arg("-C") + .arg(&self.work_dir) + .status()?; + if !status.success() { + let msg = format!("failed to extract tarball: {}", status); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + let pkg_name = input_tarball.trim_right_matches(".tar.gz"); + let pkg_name = Path::new(pkg_name).file_name().unwrap(); + let pkg_dir = Path::new(&self.work_dir).join(&pkg_name); + + // Verify the version number + let mut version = String::new(); + fs::File::open(pkg_dir.join("rust-installer-version"))? + .read_to_string(&mut version)?; + if version.trim().parse() != Ok(::RUST_INSTALLER_VERSION) { + let msg = format!("incorrect installer version in {}", input_tarball); + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + + // Copy components to new combined installer + let mut pkg_components = String::new(); + fs::File::open(pkg_dir.join("components"))? + .read_to_string(&mut pkg_components)?; + for component in pkg_components.split_whitespace() { + // All we need to do is copy the component directory + let component_dir = package_dir.join(&component); + fs::create_dir(&component_dir)?; + copy_recursive(&pkg_dir.join(&component), &component_dir)?; + + // Merge the component name + writeln!(&components, "{}", component)?; + } + } + drop(components); + + // Write the installer version + let version = fs::File::create(package_dir.join("rust-installer-version"))?; + writeln!(&version, "{}", ::RUST_INSTALLER_VERSION)?; + drop(version); + + // Copy the overlay + if !self.non_installed_overlay.is_empty() { + copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?; + } + + // Generate the install script + let output_script = package_dir.join("install.sh"); + let mut scripter = Scripter::default(); + scripter.product_name(self.product_name) + .rel_manifest_dir(self.rel_manifest_dir) + .success_message(self.success_message) + .legacy_manifest_dirs(self.legacy_manifest_dirs) + .output_script(output_script.to_str().unwrap().into()); + scripter.run()?; + + // Make the tarballs + fs::create_dir_all(&self.output_dir)?; + let output = Path::new(&self.output_dir).join(&self.package_name); + let mut tarballer = Tarballer::default(); + tarballer.work_dir(self.work_dir) + .input(self.package_name) + .output(output.to_str().unwrap().into()); + tarballer.run()?; + + Ok(()) + } +} diff --git a/src/generator.rs b/src/generator.rs index c3accc7..b5faed3 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -11,10 +11,10 @@ use std::fs; use std::io::{self, Write}; use std::path::Path; -use walkdir::WalkDir; use super::Scripter; use super::Tarballer; +use util::*; #[derive(Debug)] pub struct Generator { @@ -168,12 +168,6 @@ impl Generator { } } -/// Copies the `src` directory recursively to `dst`. Both are assumed to exist -/// when this function is called. -fn copy_recursive(src: &Path, dst: &Path) -> io::Result<()> { - copy_with_callback(src, dst, |_, _| Ok(())) -} - /// Copies the `src` directory recursively to `dst`, writing `manifest.in` too. fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> io::Result<()> { let manifest = fs::File::create(dst.join("manifest.in"))?; @@ -194,24 +188,3 @@ fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> io::Result<()> Ok(()) }) } - -/// Copies the `src` directory recursively to `dst`. Both are assumed to exist -/// when this function is called. Invokes a callback for each path visited. -fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> io::Result<()> - where F: FnMut(&Path, fs::FileType) -> io::Result<()> -{ - for entry in WalkDir::new(src).min_depth(1) { - let entry = entry?; - let file_type = entry.file_type(); - let path = entry.path().strip_prefix(src).unwrap(); - let dst = dst.join(path); - - if file_type.is_dir() { - fs::create_dir(&dst)?; - } else { - fs::copy(entry.path(), dst)?; - } - callback(&path, file_type)?; - } - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index 277fd03..4c86952 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,10 +10,13 @@ extern crate walkdir; +mod combiner; mod generator; mod scripter; mod tarballer; +mod util; +pub use combiner::Combiner; pub use generator::Generator; pub use scripter::Scripter; pub use tarballer::Tarballer; diff --git a/src/main.rs b/src/main.rs index 575e8e6..a5d8003 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ fn main() { let matches = App::from_yaml(yaml).get_matches(); match matches.subcommand() { + ("combine", Some(matches)) => combine(matches), ("generate", Some(matches)) => generate(matches), ("script", Some(matches)) => script(matches), ("tarball", Some(matches)) => tarball(matches), @@ -17,6 +18,42 @@ fn main() { } } +fn combine(matches: &ArgMatches) { + let mut com = Combiner::default(); + matches + .value_of("product-name") + .map(|s| com.product_name(s.into())); + matches + .value_of("package-name") + .map(|s| com.package_name(s.into())); + matches + .value_of("rel-manifest-dir") + .map(|s| com.rel_manifest_dir(s.into())); + matches + .value_of("success-message") + .map(|s| com.success_message(s.into())); + matches + .value_of("legacy-manifest-dirs") + .map(|s| com.legacy_manifest_dirs(s.into())); + matches + .value_of("input-tarballs") + .map(|s| com.input_tarballs(s.into())); + matches + .value_of("non-installed-overlay") + .map(|s| com.non_installed_overlay(s.into())); + matches + .value_of("work-dir") + .map(|s| com.work_dir(s.into())); + matches + .value_of("output-dir") + .map(|s| com.output_dir(s.into())); + + if let Err(e) = com.run() { + println!("failed to combine installers: {}", e); + std::process::exit(1); + } +} + fn generate(matches: &ArgMatches) { let mut gen = Generator::default(); matches diff --git a/src/main.yml b/src/main.yml index c6edebb..2f9978b 100644 --- a/src/main.yml +++ b/src/main.yml @@ -60,6 +60,54 @@ subcommands: long: output-dir takes_value: true value_name: DIR + - combine: + about: Combine installer tarballs + args: + - product-name: + help: The name of the product, for display + long: product-name + takes_value: true + value_name: NAME + - package-name: + help: The name of the package, tarball + long: package-name + takes_value: true + value_name: NAME + - rel-manifest-dir: + help: The directory under lib/ where the manifest lives + long: rel-manifest-dir + takes_value: true + value_name: DIR + - success-message: + help: The string to print after successful installation + long: success-message + takes_value: true + value_name: MESSAGE + - legacy-manifest-dirs: + help: Places to look for legacy manifests to uninstall + long: legacy-manifest-dirs + takes_value: true + value_name: DIRS + - input-tarballs: + help: Installers to combine + long: input-tarballs + takes_value: true + value_name: FILE,FILE + - non-installed-overlay: + help: Directory containing files that should not be installed + long: non-installed-overlay + takes_value: true + value_name: DIR + - work-dir: + help: The directory to do temporary work + long: work-dir + takes_value: true + value_name: DIR + - output-dir: + help: The location to put the final image and tarball + long: output-dir + takes_value: true + value_name: DIR - script: about: Generate an installation script args: diff --git a/src/tarballer.rs b/src/tarballer.rs index f642796..b1748fd 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -8,14 +8,14 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::env; -use std::ffi::{OsString, OsStr}; use std::fs; use std::io::{self, Write}; use std::path::Path; use std::process::{Command, Stdio}; use walkdir::WalkDir; +use util::*; + #[derive(Debug)] pub struct Tarballer { input: String, @@ -132,54 +132,6 @@ impl Tarballer { } } -fn get_path() -> io::Result { - let path = env::var_os("PATH").unwrap_or(OsString::new()); - // On Windows, quotes are invalid characters for filename paths, and if - // one is present as part of the PATH then that can lead to the system - // being unable to identify the files properly. See - // https://github.com/rust-lang/rust/issues/34959 for more details. - if cfg!(windows) { - if path.to_string_lossy().contains("\"") { - let msg = "PATH contains invalid character '\"'"; - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } - } - Ok(path) -} - -fn have_cmd(path: &OsStr, cmd: &str) -> bool { - for path in env::split_paths(path) { - let target = path.join(cmd); - let cmd_alt = cmd.to_string() + ".exe"; - if target.is_file() || - target.with_extension("exe").exists() || - target.join(cmd_alt).exists() { - return true; - } - } - false -} - -fn need_cmd(path: &OsStr, cmd: &str) -> io::Result<()> { - if have_cmd(path, cmd) { - Ok(()) - } else { - let msg = format!("couldn't find required command: '{}'", cmd); - Err(io::Error::new(io::ErrorKind::NotFound, msg)) - } -} - -fn need_either_cmd(path: &OsStr, cmd1: &str, cmd2: &str) -> io::Result { - if have_cmd(path, cmd1) { - Ok(true) - } else if have_cmd(path, cmd2) { - Ok(false) - } else { - let msg = format!("couldn't find either command: '{}' or '{}'", cmd1, cmd2); - Err(io::Error::new(io::ErrorKind::NotFound, msg)) - } -} - fn get_recursive_paths(root: &Path, name: &Path) -> io::Result> { let mut paths = vec![]; for entry in WalkDir::new(root.join(name)).min_depth(1) { diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..72da914 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,91 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::env; +use std::ffi::{OsString, OsStr}; +use std::fs; +use std::io; +use std::path::Path; +use walkdir::WalkDir; + +pub fn get_path() -> io::Result { + let path = env::var_os("PATH").unwrap_or(OsString::new()); + // On Windows, quotes are invalid characters for filename paths, and if + // one is present as part of the PATH then that can lead to the system + // being unable to identify the files properly. See + // https://github.com/rust-lang/rust/issues/34959 for more details. + if cfg!(windows) { + if path.to_string_lossy().contains("\"") { + let msg = "PATH contains invalid character '\"'"; + return Err(io::Error::new(io::ErrorKind::Other, msg)); + } + } + Ok(path) +} + +pub fn have_cmd(path: &OsStr, cmd: &str) -> bool { + for path in env::split_paths(path) { + let target = path.join(cmd); + let cmd_alt = cmd.to_string() + ".exe"; + if target.is_file() || + target.with_extension("exe").exists() || + target.join(cmd_alt).exists() { + return true; + } + } + false +} + +pub fn need_cmd(path: &OsStr, cmd: &str) -> io::Result<()> { + if have_cmd(path, cmd) { + Ok(()) + } else { + let msg = format!("couldn't find required command: '{}'", cmd); + Err(io::Error::new(io::ErrorKind::NotFound, msg)) + } +} + +pub fn need_either_cmd(path: &OsStr, cmd1: &str, cmd2: &str) -> io::Result { + if have_cmd(path, cmd1) { + Ok(true) + } else if have_cmd(path, cmd2) { + Ok(false) + } else { + let msg = format!("couldn't find either command: '{}' or '{}'", cmd1, cmd2); + Err(io::Error::new(io::ErrorKind::NotFound, msg)) + } +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. +pub fn copy_recursive(src: &Path, dst: &Path) -> io::Result<()> { + copy_with_callback(src, dst, |_, _| Ok(())) +} + +/// Copies the `src` directory recursively to `dst`. Both are assumed to exist +/// when this function is called. Invokes a callback for each path visited. +pub fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> io::Result<()> + where F: FnMut(&Path, fs::FileType) -> io::Result<()> +{ + for entry in WalkDir::new(src).min_depth(1) { + let entry = entry?; + let file_type = entry.file_type(); + let path = entry.path().strip_prefix(src).unwrap(); + let dst = dst.join(path); + + if file_type.is_dir() { + fs::create_dir(&dst)?; + } else { + fs::copy(entry.path(), dst)?; + } + callback(&path, file_type)?; + } + Ok(()) +} From d337855979345bc54a640f020c14a895685c902d Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 14:57:16 -0700 Subject: [PATCH 06/21] Disable binary docs --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index a1521e4..472596b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,8 @@ walkdir = "1.0.7" [dependencies.clap] features = ["yaml"] version = "2.22.1" + +[[bin]] +name = "rust-installer" +path = "src/main.rs" +doc = false From 4ad66b2649b58a96531b85ecb21c04df5762afd0 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 15:46:44 -0700 Subject: [PATCH 07/21] Use a macro to reduce struct boilerplate --- src/combiner.rs | 101 ++++++++++------------------------------ src/generator.rs | 119 ++++++++++++----------------------------------- src/lib.rs | 4 +- src/main.rs | 56 +++++++++++----------- src/scripter.rs | 60 ++++++------------------ src/tarballer.rs | 41 ++++------------ src/util.rs | 29 ++++++++++++ 7 files changed, 139 insertions(+), 271 deletions(-) diff --git a/src/combiner.rs b/src/combiner.rs index 20e0ec2..716299b 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -17,90 +17,39 @@ use super::Scripter; use super::Tarballer; use util::*; -#[derive(Debug)] -pub struct Combiner { - product_name: String, - package_name: String, - rel_manifest_dir: String, - success_message: String, - legacy_manifest_dirs: String, - input_tarballs: String, - non_installed_overlay: String, - work_dir: String, - output_dir: String, -} +actor!{ + #[derive(Debug)] + pub struct Combiner { + /// The name of the product, for display + product_name: String = "Product", -impl Default for Combiner { - fn default() -> Combiner { - Combiner { - product_name: "Product".into(), - package_name: "package".into(), - rel_manifest_dir: "packagelib".into(), - success_message: "Installed.".into(), - legacy_manifest_dirs: "".into(), - input_tarballs: "".into(), - non_installed_overlay: "".into(), - work_dir: "./workdir".into(), - output_dir: "./dist".into(), - } - } -} + /// The name of the package, tarball + package_name: String = "package", -impl Combiner { - /// The name of the product, for display - pub fn product_name(&mut self, value: String) -> &mut Self { - self.product_name = value; - self - } + /// The directory under lib/ where the manifest lives + rel_manifest_dir: String = "packagelib", - /// The name of the package, tarball - pub fn package_name(&mut self, value: String) -> &mut Self { - self.package_name = value; - self - } + /// The string to print after successful installation + success_message: String = "Installed.", - /// The directory under lib/ where the manifest lives - pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { - self.rel_manifest_dir = value; - self - } + /// Places to look for legacy manifests to uninstall + legacy_manifest_dirs: String = "", - /// The string to print after successful installation - pub fn success_message(&mut self, value: String) -> &mut Self { - self.success_message = value; - self - } + /// Installers to combine + input_tarballs: String = "", - /// Places to look for legacy manifests to uninstall - pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { - self.legacy_manifest_dirs = value; - self - } - - /// Installers to combine - pub fn input_tarballs(&mut self, value: String) -> &mut Self { - self.input_tarballs = value; - self - } - - /// Directory containing files that should not be installed - pub fn non_installed_overlay(&mut self, value: String) -> &mut Self { - self.non_installed_overlay = value; - self - } + /// Directory containing files that should not be installed + non_installed_overlay: String = "", - /// The directory to do temporary work - pub fn work_dir(&mut self, value: String) -> &mut Self { - self.work_dir = value; - self - } + /// The directory to do temporary work + work_dir: String = "./workdir", - /// The location to put the final image and tarball - pub fn output_dir(&mut self, value: String) -> &mut Self { - self.output_dir = value; - self + /// The location to put the final image and tarball + output_dir: String = "./dist", } +} +impl Combiner { /// Generate the actual installer tarball pub fn run(self) -> io::Result<()> { let path = get_path()?; @@ -175,7 +124,7 @@ impl Combiner { .rel_manifest_dir(self.rel_manifest_dir) .success_message(self.success_message) .legacy_manifest_dirs(self.legacy_manifest_dirs) - .output_script(output_script.to_str().unwrap().into()); + .output_script(output_script.to_str().unwrap()); scripter.run()?; // Make the tarballs @@ -184,7 +133,7 @@ impl Combiner { let mut tarballer = Tarballer::default(); tarballer.work_dir(self.work_dir) .input(self.package_name) - .output(output.to_str().unwrap().into()); + .output(output.to_str().unwrap()); tarballer.run()?; Ok(()) diff --git a/src/generator.rs b/src/generator.rs index b5faed3..12462a7 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -16,106 +16,45 @@ use super::Scripter; use super::Tarballer; use util::*; -#[derive(Debug)] -pub struct Generator { - product_name: String, - component_name: String, - package_name: String, - rel_manifest_dir: String, - success_message: String, - legacy_manifest_dirs: String, - non_installed_overlay: String, - bulk_dirs: String, - image_dir: String, - work_dir: String, - output_dir: String, -} +actor!{ + #[derive(Debug)] + pub struct Generator { + /// The name of the product, for display + product_name: String = "Product", -impl Default for Generator { - fn default() -> Generator { - Generator { - product_name: "Product".into(), - component_name: "component".into(), - package_name: "package".into(), - rel_manifest_dir: "packagelib".into(), - success_message: "Installed.".into(), - legacy_manifest_dirs: "".into(), - non_installed_overlay: "".into(), - bulk_dirs: "".into(), - image_dir: "./install_image".into(), - work_dir: "./workdir".into(), - output_dir: "./dist".into(), - } - } -} + /// The name of the component, distinct from other installed components + component_name: String = "component", -impl Generator { - /// The name of the product, for display - pub fn product_name(&mut self, value: String) -> &mut Self { - self.product_name = value; - self - } + /// The name of the package, tarball + package_name: String = "package", - /// The name of the component, distinct from other installed components - pub fn component_name(&mut self, value: String) -> &mut Self { - self.component_name = value; - self - } + /// The directory under lib/ where the manifest lives + rel_manifest_dir: String = "packagelib", - /// The name of the package, tarball - pub fn package_name(&mut self, value: String) -> &mut Self { - self.package_name = value; - self - } + /// The string to print after successful installation + success_message: String = "Installed.", - /// The directory under lib/ where the manifest lives - pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { - self.rel_manifest_dir = value; - self - } + /// Places to look for legacy manifests to uninstall + legacy_manifest_dirs: String = "", - /// The string to print after successful installation - pub fn success_message(&mut self, value: String) -> &mut Self { - self.success_message = value; - self - } + /// Directory containing files that should not be installed + non_installed_overlay: String = "", - /// Places to look for legacy manifests to uninstall - pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { - self.legacy_manifest_dirs = value; - self - } + /// Path prefixes of directories that should be installed/uninstalled in bulk + bulk_dirs: String = "", - /// Directory containing files that should not be installed - pub fn non_installed_overlay(&mut self, value: String) -> &mut Self { - self.non_installed_overlay = value; - self - } + /// The directory containing the installation medium + image_dir: String = "./install_image", - /// Path prefixes of directories that should be installed/uninstalled in bulk - pub fn bulk_dirs(&mut self, value: String) -> &mut Self { - self.bulk_dirs = value; - self - } - - /// The directory containing the installation medium - pub fn image_dir(&mut self, value: String) -> &mut Self { - self.image_dir = value; - self - } + /// The directory to do temporary work + work_dir: String = "./workdir", - /// The directory to do temporary work - pub fn work_dir(&mut self, value: String) -> &mut Self { - self.work_dir = value; - self - } - - /// The location to put the final image and tarball - pub fn output_dir(&mut self, value: String) -> &mut Self { - self.output_dir = value; - self + /// The location to put the final image and tarball + output_dir: String = "./dist", } +} +impl Generator { /// Generate the actual installer tarball pub fn run(self) -> io::Result<()> { fs::create_dir_all(&self.work_dir)?; @@ -152,7 +91,7 @@ impl Generator { .rel_manifest_dir(self.rel_manifest_dir) .success_message(self.success_message) .legacy_manifest_dirs(self.legacy_manifest_dirs) - .output_script(output_script.to_str().unwrap().into()); + .output_script(output_script.to_str().unwrap()); scripter.run()?; // Make the tarballs @@ -161,7 +100,7 @@ impl Generator { let mut tarballer = Tarballer::default(); tarballer.work_dir(self.work_dir) .input(self.package_name) - .output(output.to_str().unwrap().into()); + .output(output.to_str().unwrap()); tarballer.run()?; Ok(()) diff --git a/src/lib.rs b/src/lib.rs index 4c86952..d926613 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,11 +10,13 @@ extern crate walkdir; +#[macro_use] +mod util; + mod combiner; mod generator; mod scripter; mod tarballer; -mod util; pub use combiner::Combiner; pub use generator::Generator; diff --git a/src/main.rs b/src/main.rs index a5d8003..0eef185 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,31 +22,31 @@ fn combine(matches: &ArgMatches) { let mut com = Combiner::default(); matches .value_of("product-name") - .map(|s| com.product_name(s.into())); + .map(|s| com.product_name(s)); matches .value_of("package-name") - .map(|s| com.package_name(s.into())); + .map(|s| com.package_name(s)); matches .value_of("rel-manifest-dir") - .map(|s| com.rel_manifest_dir(s.into())); + .map(|s| com.rel_manifest_dir(s)); matches .value_of("success-message") - .map(|s| com.success_message(s.into())); + .map(|s| com.success_message(s)); matches .value_of("legacy-manifest-dirs") - .map(|s| com.legacy_manifest_dirs(s.into())); + .map(|s| com.legacy_manifest_dirs(s)); matches .value_of("input-tarballs") - .map(|s| com.input_tarballs(s.into())); + .map(|s| com.input_tarballs(s)); matches .value_of("non-installed-overlay") - .map(|s| com.non_installed_overlay(s.into())); + .map(|s| com.non_installed_overlay(s)); matches .value_of("work-dir") - .map(|s| com.work_dir(s.into())); + .map(|s| com.work_dir(s)); matches .value_of("output-dir") - .map(|s| com.output_dir(s.into())); + .map(|s| com.output_dir(s)); if let Err(e) = com.run() { println!("failed to combine installers: {}", e); @@ -58,37 +58,37 @@ fn generate(matches: &ArgMatches) { let mut gen = Generator::default(); matches .value_of("product-name") - .map(|s| gen.product_name(s.into())); + .map(|s| gen.product_name(s)); matches .value_of("component-name") - .map(|s| gen.component_name(s.into())); + .map(|s| gen.component_name(s)); matches .value_of("package-name") - .map(|s| gen.package_name(s.into())); + .map(|s| gen.package_name(s)); matches .value_of("rel-manifest-dir") - .map(|s| gen.rel_manifest_dir(s.into())); + .map(|s| gen.rel_manifest_dir(s)); matches .value_of("success-message") - .map(|s| gen.success_message(s.into())); + .map(|s| gen.success_message(s)); matches .value_of("legacy-manifest-dirs") - .map(|s| gen.legacy_manifest_dirs(s.into())); + .map(|s| gen.legacy_manifest_dirs(s)); matches .value_of("non-installed-overlay") - .map(|s| gen.non_installed_overlay(s.into())); + .map(|s| gen.non_installed_overlay(s)); matches .value_of("bulk-dirs") - .map(|s| gen.bulk_dirs(s.into())); + .map(|s| gen.bulk_dirs(s)); matches .value_of("image-dir") - .map(|s| gen.image_dir(s.into())); + .map(|s| gen.image_dir(s)); matches .value_of("work-dir") - .map(|s| gen.work_dir(s.into())); + .map(|s| gen.work_dir(s)); matches .value_of("output-dir") - .map(|s| gen.output_dir(s.into())); + .map(|s| gen.output_dir(s)); if let Err(e) = gen.run() { println!("failed to generate installer: {}", e); @@ -100,19 +100,19 @@ fn script(matches: &ArgMatches) { let mut scr = Scripter::default(); matches .value_of("product-name") - .map(|s| scr.product_name(s.into())); + .map(|s| scr.product_name(s)); matches .value_of("rel-manifest-dir") - .map(|s| scr.rel_manifest_dir(s.into())); + .map(|s| scr.rel_manifest_dir(s)); matches .value_of("success-message") - .map(|s| scr.success_message(s.into())); + .map(|s| scr.success_message(s)); matches .value_of("legacy-manifest-dirs") - .map(|s| scr.legacy_manifest_dirs(s.into())); + .map(|s| scr.legacy_manifest_dirs(s)); matches .value_of("output-script") - .map(|s| scr.output_script(s.into())); + .map(|s| scr.output_script(s)); if let Err(e) = scr.run() { println!("failed to generate installation script: {}", e); @@ -124,13 +124,13 @@ fn tarball(matches: &ArgMatches) { let mut tar = Tarballer::default(); matches .value_of("input") - .map(|s| tar.input(s.into())); + .map(|s| tar.input(s)); matches .value_of("output") - .map(|s| tar.output(s.into())); + .map(|s| tar.output(s)); matches .value_of("work-dir") - .map(|s| tar.work_dir(s.into())); + .map(|s| tar.work_dir(s)); if let Err(e) = tar.run() { println!("failed to generate tarballs: {}", e); diff --git a/src/scripter.rs b/src/scripter.rs index a5b51ba..76c9fe0 100644 --- a/src/scripter.rs +++ b/src/scripter.rs @@ -18,58 +18,28 @@ use std::os::unix::fs::OpenOptionsExt; const TEMPLATE: &'static str = include_str!("../install-template.sh"); -#[derive(Debug)] -pub struct Scripter { - product_name: String, - rel_manifest_dir: String, - success_message: String, - legacy_manifest_dirs: String, - output_script: String, -} - -impl Default for Scripter { - fn default() -> Scripter { - Scripter { - product_name: "Product".into(), - rel_manifest_dir: "manifestlib".into(), - success_message: "Installed.".into(), - legacy_manifest_dirs: "".into(), - output_script: "install.sh".into(), - } - } -} -impl Scripter { - /// The name of the product, for display - pub fn product_name(&mut self, value: String) -> &mut Self { - self.product_name = value; - self - } +actor!{ + #[derive(Debug)] + pub struct Scripter { + /// The name of the product, for display + product_name: String = "Product", - /// The directory under lib/ where the manifest lives - pub fn rel_manifest_dir(&mut self, value: String) -> &mut Self { - self.rel_manifest_dir = value; - self - } + /// The directory under lib/ where the manifest lives + rel_manifest_dir: String = "manifestlib", - /// The string to print after successful installation - pub fn success_message(&mut self, value: String) -> &mut Self { - self.success_message = value; - self - } + /// The string to print after successful installation + success_message: String = "Installed.", - /// Places to look for legacy manifests to uninstall - pub fn legacy_manifest_dirs(&mut self, value: String) -> &mut Self { - self.legacy_manifest_dirs = value; - self - } + /// Places to look for legacy manifests to uninstall + legacy_manifest_dirs: String = "", - /// The name of the output script - pub fn output_script(&mut self, value: String) -> &mut Self { - self.output_script = value; - self + /// The name of the output script + output_script: String = "install.sh", } +} +impl Scripter { /// Generate the actual installer script pub fn run(self) -> io::Result<()> { // Replace dashes in the success message with spaces (our arg handling botches spaces) diff --git a/src/tarballer.rs b/src/tarballer.rs index b1748fd..72037b4 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -16,42 +16,21 @@ use walkdir::WalkDir; use util::*; -#[derive(Debug)] -pub struct Tarballer { - input: String, - output: String, - work_dir: String, -} +actor!{ + #[derive(Debug)] + pub struct Tarballer { + /// The input folder to be compressed + input: String = "package", -impl Default for Tarballer { - fn default() -> Tarballer { - Tarballer { - input: "package".into(), - output: "./dist".into(), - work_dir: "./workdir.".into(), - } + /// The prefix of the tarballs + output: String = "./dist", + + /// The fold in which the input is to be found + work_dir: String = "./workdir", } } impl Tarballer { - /// The input folder to be compressed - pub fn input(&mut self, value: String) -> &mut Self { - self.input = value; - self - } - - /// The prefix of the tarballs - pub fn output(&mut self, value: String) -> &mut Self { - self.output = value; - self - } - - /// The fold in which the input is to be found - pub fn work_dir(&mut self, value: String) -> &mut Self { - self.work_dir = value; - self - } - /// Generate the actual tarballs pub fn run(self) -> io::Result<()> { let path = get_path()?; diff --git a/src/util.rs b/src/util.rs index 72da914..4c42da1 100644 --- a/src/util.rs +++ b/src/util.rs @@ -89,3 +89,32 @@ pub fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> io::Res } Ok(()) } + + +/// Create an "actor" with default values and setters for all fields. +macro_rules! actor { + ($( #[ $attr:meta ] )+ pub struct $name:ident { + $( $( #[ $field_attr:meta ] )+ $field:ident : $type:ty = $default:expr, )* + }) => { + $( #[ $attr ] )+ + pub struct $name { + $( $( #[ $field_attr ] )+ $field : $type, )* + } + + impl Default for $name { + fn default() -> Self { + $name { + $( $field : $default.into(), )* + } + } + } + + impl $name { + $( $( #[ $field_attr ] )+ + pub fn $field>(&mut self, value: T) -> &mut Self { + self.$field = value.into(); + self + })+ + } + } +} From e72663f8c129f706f51ce50fc6c4b90db89ea61f Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 16:03:05 -0700 Subject: [PATCH 08/21] Use a macro to reduce clap boilerplate a little --- src/main.rs | 143 +++++++++++++++++++--------------------------------- 1 file changed, 51 insertions(+), 92 deletions(-) diff --git a/src/main.rs b/src/main.rs index 0eef185..882aa57 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,121 +18,80 @@ fn main() { } } +/// Parse clap arguements into the type constructor. +macro_rules! parse( + ($matches:expr => $type:ty { $( $option:tt => $setter:ident, )* }) => { + { + let mut command: $type = Default::default(); + $( $matches.value_of($option).map(|s| command.$setter(s)); )* + command + } + } +); + fn combine(matches: &ArgMatches) { - let mut com = Combiner::default(); - matches - .value_of("product-name") - .map(|s| com.product_name(s)); - matches - .value_of("package-name") - .map(|s| com.package_name(s)); - matches - .value_of("rel-manifest-dir") - .map(|s| com.rel_manifest_dir(s)); - matches - .value_of("success-message") - .map(|s| com.success_message(s)); - matches - .value_of("legacy-manifest-dirs") - .map(|s| com.legacy_manifest_dirs(s)); - matches - .value_of("input-tarballs") - .map(|s| com.input_tarballs(s)); - matches - .value_of("non-installed-overlay") - .map(|s| com.non_installed_overlay(s)); - matches - .value_of("work-dir") - .map(|s| com.work_dir(s)); - matches - .value_of("output-dir") - .map(|s| com.output_dir(s)); + let combiner = parse!(matches => Combiner { + "product-name" => product_name, + "package-name" => package_name, + "rel-manifest-dir" => rel_manifest_dir, + "success-message" => success_message, + "legacy-manifest-dirs" => legacy_manifest_dirs, + "input-tarballs" => input_tarballs, + "non-installed-overlay" => non_installed_overlay, + "work-dir" => work_dir, + "output-dir" => output_dir, + }); - if let Err(e) = com.run() { + if let Err(e) = combiner.run() { println!("failed to combine installers: {}", e); std::process::exit(1); } } fn generate(matches: &ArgMatches) { - let mut gen = Generator::default(); - matches - .value_of("product-name") - .map(|s| gen.product_name(s)); - matches - .value_of("component-name") - .map(|s| gen.component_name(s)); - matches - .value_of("package-name") - .map(|s| gen.package_name(s)); - matches - .value_of("rel-manifest-dir") - .map(|s| gen.rel_manifest_dir(s)); - matches - .value_of("success-message") - .map(|s| gen.success_message(s)); - matches - .value_of("legacy-manifest-dirs") - .map(|s| gen.legacy_manifest_dirs(s)); - matches - .value_of("non-installed-overlay") - .map(|s| gen.non_installed_overlay(s)); - matches - .value_of("bulk-dirs") - .map(|s| gen.bulk_dirs(s)); - matches - .value_of("image-dir") - .map(|s| gen.image_dir(s)); - matches - .value_of("work-dir") - .map(|s| gen.work_dir(s)); - matches - .value_of("output-dir") - .map(|s| gen.output_dir(s)); + let generator = parse!(matches => Generator { + "product-name" => product_name, + "component-name" => component_name, + "package-name" => package_name, + "rel-manifest-dir" => rel_manifest_dir, + "success-message" => success_message, + "legacy-manifest-dirs" => legacy_manifest_dirs, + "non-installed-overlay" => non_installed_overlay, + "bulk-dirs" => bulk_dirs, + "image-dir" => image_dir, + "work-dir" => work_dir, + "output-dir" => output_dir, + }); - if let Err(e) = gen.run() { + if let Err(e) = generator.run() { println!("failed to generate installer: {}", e); std::process::exit(1); } } fn script(matches: &ArgMatches) { - let mut scr = Scripter::default(); - matches - .value_of("product-name") - .map(|s| scr.product_name(s)); - matches - .value_of("rel-manifest-dir") - .map(|s| scr.rel_manifest_dir(s)); - matches - .value_of("success-message") - .map(|s| scr.success_message(s)); - matches - .value_of("legacy-manifest-dirs") - .map(|s| scr.legacy_manifest_dirs(s)); - matches - .value_of("output-script") - .map(|s| scr.output_script(s)); + let scripter = parse!(matches => Scripter { + "product-name" => product_name, + "rel-manifest-dir" => rel_manifest_dir, + "success-message" => success_message, + "legacy-manifest-dirs" => legacy_manifest_dirs, + "output-script" => output_script, + }); - if let Err(e) = scr.run() { + if let Err(e) = scripter.run() { println!("failed to generate installation script: {}", e); std::process::exit(1); } } fn tarball(matches: &ArgMatches) { - let mut tar = Tarballer::default(); - matches - .value_of("input") - .map(|s| tar.input(s)); - matches - .value_of("output") - .map(|s| tar.output(s)); - matches - .value_of("work-dir") - .map(|s| tar.work_dir(s)); + let tarballer = parse!(matches => Tarballer { + "input" => input, + "output" => output, + "work-dir" => work_dir, + }); - if let Err(e) = tar.run() { + if let Err(e) = tarballer.run() { println!("failed to generate tarballs: {}", e); std::process::exit(1); } From a0c9a404f318bce2fd179be83c157904a0ef1181 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 16:27:22 -0700 Subject: [PATCH 09/21] Use the flate2 and tar crates for combining --- Cargo.toml | 12 +++++++----- src/combiner.rs | 21 ++++++--------------- src/lib.rs | 2 ++ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 472596b..10b0374 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,14 +3,16 @@ authors = ["The Rust Project Developers"] name = "installer" version = "0.0.0" +[[bin]] +doc = false +name = "rust-installer" +path = "src/main.rs" + [dependencies] +flate2 = "0.2.19" +tar = "0.4.11" walkdir = "1.0.7" [dependencies.clap] features = ["yaml"] version = "2.22.1" - -[[bin]] -name = "rust-installer" -path = "src/main.rs" -doc = false diff --git a/src/combiner.rs b/src/combiner.rs index 716299b..81e7c47 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -11,7 +11,8 @@ use std::fs; use std::io::{self, Read, Write}; use std::path::Path; -use std::process::Command; +use flate2::read::GzDecoder; +use tar::Archive; use super::Scripter; use super::Tarballer; @@ -50,11 +51,8 @@ actor!{ } impl Combiner { - /// Generate the actual installer tarball + /// Combine the installer tarballs pub fn run(self) -> io::Result<()> { - let path = get_path()?; - need_cmd(&path, "tar")?; - fs::create_dir_all(&self.work_dir)?; let package_dir = Path::new(&self.work_dir).join(&self.package_name); @@ -67,16 +65,9 @@ impl Combiner { let components = fs::File::create(package_dir.join("components"))?; for input_tarball in self.input_tarballs.split(',').map(str::trim).filter(|s| !s.is_empty()) { // Extract the input tarballs - let status = Command::new("tar") - .arg("xzf") - .arg(&input_tarball) - .arg("-C") - .arg(&self.work_dir) - .status()?; - if !status.success() { - let msg = format!("failed to extract tarball: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + let input = fs::File::open(&input_tarball)?; + let deflated = GzDecoder::new(input)?; + Archive::new(deflated).unpack(&self.work_dir)?; let pkg_name = input_tarball.trim_right_matches(".tar.gz"); let pkg_name = Path::new(pkg_name).file_name().unwrap(); diff --git a/src/lib.rs b/src/lib.rs index d926613..7195142 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +extern crate flate2; +extern crate tar; extern crate walkdir; #[macro_use] From 4eae6a8c7243c6c74e970ccaaa8f94476b438f37 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 16:41:51 -0700 Subject: [PATCH 10/21] Use the tar crate for creating tarballs --- src/tarballer.rs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index 72037b4..414466f 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -9,9 +9,11 @@ // except according to those terms. use std::fs; -use std::io::{self, Write}; +use std::io; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::Command; + +use tar::Builder; use walkdir::WalkDir; use util::*; @@ -34,7 +36,6 @@ impl Tarballer { /// Generate the actual tarballs pub fn run(self) -> io::Result<()> { let path = get_path()?; - need_cmd(&path, "tar")?; need_cmd(&path, "gzip")?; let have_xz = need_either_cmd(&path, "xz", "7z")?; @@ -56,24 +57,19 @@ impl Tarballer { paths.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); // Write the tar file - let mut child = Command::new("tar") - .arg("-cf") - .arg(&tar) - .arg("-T") - .arg("-") - .stdin(Stdio::piped()) - .current_dir(&self.work_dir) - .spawn()?; - if let Some(stdin) = child.stdin.as_mut() { - for path in paths { - writeln!(stdin, "{}", path)?; + let output = fs::File::create(&tar)?; + let mut builder = Builder::new(output); + for path in paths { + let path = Path::new(&path); + let src = Path::new(&self.work_dir).join(path); + if path.is_dir() { + builder.append_dir(path, src)?; + } else { + let mut src = fs::File::open(src)?; + builder.append_file(path, &mut src)?; } } - let status = child.wait()?; - if !status.success() { - let msg = format!("failed to make tarball: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + builder.into_inner()?; // Write the .tar.xz file let status = if have_xz { From 30dd001864fb30df9834e9009e8440c0af054ab2 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 16:57:26 -0700 Subject: [PATCH 11/21] Use the flate2 crate for creating gzip tarballs --- src/tarballer.rs | 19 +++++++++++-------- src/util.rs | 2 ++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index 414466f..d03d3f0 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -13,6 +13,8 @@ use std::io; use std::path::Path; use std::process::Command; +use flate2; +use flate2::write::GzEncoder; use tar::Builder; use walkdir::WalkDir; @@ -36,7 +38,6 @@ impl Tarballer { /// Generate the actual tarballs pub fn run(self) -> io::Result<()> { let path = get_path()?; - need_cmd(&path, "gzip")?; let have_xz = need_either_cmd(&path, "xz", "7z")?; let tar = self.output + ".tar"; @@ -95,13 +96,15 @@ impl Tarballer { } // Write the .tar.gz file (removing the .tar) - let status = Command::new("gzip") - .arg(&tar) - .status()?; - if !status.success() { - let msg = format!("failed to make tar.gz: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + let mut input = fs::File::open(&tar)?; + let output = fs::File::create(&tar_gz)?; + let mut encoded = GzEncoder::new(output, flate2::Compression::Best); + io::copy(&mut input, &mut encoded)?; + encoded.finish()?; + drop(input); + + // Remove the .tar file + fs::remove_file(&tar)?; Ok(()) } diff --git a/src/util.rs b/src/util.rs index 4c42da1..3f19194 100644 --- a/src/util.rs +++ b/src/util.rs @@ -8,6 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. + use std::env; use std::ffi::{OsString, OsStr}; use std::fs; @@ -43,6 +44,7 @@ pub fn have_cmd(path: &OsStr, cmd: &str) -> bool { false } +#[allow(dead_code)] pub fn need_cmd(path: &OsStr, cmd: &str) -> io::Result<()> { if have_cmd(path, cmd) { Ok(()) From 5dd4246757b04c7e21b3c417fd9ef8ba1b32ac1e Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 17:09:46 -0700 Subject: [PATCH 12/21] Fix cfg for calling OpenOptionsExt::mode --- src/scripter.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scripter.rs b/src/scripter.rs index 76c9fe0..f066320 100644 --- a/src/scripter.rs +++ b/src/scripter.rs @@ -59,9 +59,7 @@ impl Scripter { let mut options = fs::OpenOptions::new(); options.write(true).create_new(true); - if cfg!(unix) { - options.mode(0o755); - } + #[cfg(unix)] options.mode(0o755); let output = options.open(self.output_script)?; writeln!(&output, "{}", script) } From 1df311d4ea8a482e028f48fe1f207a2f549b9829 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 18:09:43 -0700 Subject: [PATCH 13/21] Use the xz2 crate for creating xz tarballs --- Cargo.toml | 1 + src/lib.rs | 1 + src/tarballer.rs | 46 ++++++++++++------------------------------- src/util.rs | 51 ------------------------------------------------ 4 files changed, 15 insertions(+), 84 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 10b0374..c3d8f9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" flate2 = "0.2.19" tar = "0.4.11" walkdir = "1.0.7" +xz2 = "0.1.3" [dependencies.clap] features = ["yaml"] diff --git a/src/lib.rs b/src/lib.rs index 7195142..885461c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ extern crate flate2; extern crate tar; extern crate walkdir; +extern crate xz2; #[macro_use] mod util; diff --git a/src/tarballer.rs b/src/tarballer.rs index d03d3f0..548e136 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -9,16 +9,14 @@ // except according to those terms. use std::fs; -use std::io; +use std::io::{self, Seek, SeekFrom}; use std::path::Path; -use std::process::Command; use flate2; use flate2::write::GzEncoder; use tar::Builder; use walkdir::WalkDir; - -use util::*; +use xz2::write::XzEncoder; actor!{ #[derive(Debug)] @@ -37,9 +35,6 @@ actor!{ impl Tarballer { /// Generate the actual tarballs pub fn run(self) -> io::Result<()> { - let path = get_path()?; - let have_xz = need_either_cmd(&path, "xz", "7z")?; - let tar = self.output + ".tar"; let tar_gz = tar.clone() + ".gz"; let tar_xz = tar.clone() + ".xz"; @@ -58,7 +53,8 @@ impl Tarballer { paths.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); // Write the tar file - let output = fs::File::create(&tar)?; + // let output = fs::File::create(&tar)?; + let output = fs::OpenOptions::new().read(true).write(true).create_new(true).open(&tar)?; let mut builder = Builder::new(output); for path in paths { let path = Path::new(&path); @@ -70,40 +66,24 @@ impl Tarballer { builder.append_file(path, &mut src)?; } } - builder.into_inner()?; + let mut input = builder.into_inner()?; // Write the .tar.xz file - let status = if have_xz { - Command::new("xz") - .arg("-9") - .arg("--keep") - .arg(&tar) - .status()? - } else { - Command::new("7z") - .arg("a") - .arg("-bd") - .arg("-txz") - .arg("-mx=9") - .arg("-mmt=off") - .arg(&tar_xz) - .arg(&tar) - .status()? - }; - if !status.success() { - let msg = format!("failed to make tar.xz: {}", status); - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } + let output = fs::File::create(&tar_xz)?; + let mut encoded = XzEncoder::new(output, 9); + input.seek(SeekFrom::Start(0))?; + io::copy(&mut input, &mut encoded)?; + encoded.finish()?; - // Write the .tar.gz file (removing the .tar) - let mut input = fs::File::open(&tar)?; + // Write the .tar.gz file let output = fs::File::create(&tar_gz)?; let mut encoded = GzEncoder::new(output, flate2::Compression::Best); + input.seek(SeekFrom::Start(0))?; io::copy(&mut input, &mut encoded)?; encoded.finish()?; - drop(input); // Remove the .tar file + drop(input); fs::remove_file(&tar)?; Ok(()) diff --git a/src/util.rs b/src/util.rs index 3f19194..f086369 100644 --- a/src/util.rs +++ b/src/util.rs @@ -9,62 +9,11 @@ // except according to those terms. -use std::env; -use std::ffi::{OsString, OsStr}; use std::fs; use std::io; use std::path::Path; use walkdir::WalkDir; -pub fn get_path() -> io::Result { - let path = env::var_os("PATH").unwrap_or(OsString::new()); - // On Windows, quotes are invalid characters for filename paths, and if - // one is present as part of the PATH then that can lead to the system - // being unable to identify the files properly. See - // https://github.com/rust-lang/rust/issues/34959 for more details. - if cfg!(windows) { - if path.to_string_lossy().contains("\"") { - let msg = "PATH contains invalid character '\"'"; - return Err(io::Error::new(io::ErrorKind::Other, msg)); - } - } - Ok(path) -} - -pub fn have_cmd(path: &OsStr, cmd: &str) -> bool { - for path in env::split_paths(path) { - let target = path.join(cmd); - let cmd_alt = cmd.to_string() + ".exe"; - if target.is_file() || - target.with_extension("exe").exists() || - target.join(cmd_alt).exists() { - return true; - } - } - false -} - -#[allow(dead_code)] -pub fn need_cmd(path: &OsStr, cmd: &str) -> io::Result<()> { - if have_cmd(path, cmd) { - Ok(()) - } else { - let msg = format!("couldn't find required command: '{}'", cmd); - Err(io::Error::new(io::ErrorKind::NotFound, msg)) - } -} - -pub fn need_either_cmd(path: &OsStr, cmd1: &str, cmd2: &str) -> io::Result { - if have_cmd(path, cmd1) { - Ok(true) - } else if have_cmd(path, cmd2) { - Ok(false) - } else { - let msg = format!("couldn't find either command: '{}' or '{}'", cmd1, cmd2); - Err(io::Error::new(io::ErrorKind::NotFound, msg)) - } -} - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. pub fn copy_recursive(src: &Path, dst: &Path) -> io::Result<()> { From fbac639d55c809b1c2699fb19a174bbf0cfdf146 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 4 May 2017 18:32:09 -0700 Subject: [PATCH 14/21] Write tarballs without an intermediate uncompressed file --- src/tarballer.rs | 59 ++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index 548e136..eaa4573 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -9,7 +9,7 @@ // except according to those terms. use std::fs; -use std::io::{self, Seek, SeekFrom}; +use std::io::{self, Write}; use std::path::Path; use flate2; @@ -35,12 +35,11 @@ actor!{ impl Tarballer { /// Generate the actual tarballs pub fn run(self) -> io::Result<()> { - let tar = self.output + ".tar"; - let tar_gz = tar.clone() + ".gz"; - let tar_xz = tar.clone() + ".xz"; + let tar_gz = self.output.clone() + ".tar.gz"; + let tar_xz = self.output.clone() + ".tar.xz"; // Remove any existing files - for file in &[&tar, &tar_gz, &tar_xz] { + for file in &[&tar_gz, &tar_xz] { if Path::new(file).exists() { fs::remove_file(file)?; } @@ -52,10 +51,16 @@ impl Tarballer { let mut paths = get_recursive_paths(self.work_dir.as_ref(), self.input.as_ref())?; paths.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); - // Write the tar file - // let output = fs::File::create(&tar)?; - let output = fs::OpenOptions::new().read(true).write(true).create_new(true).open(&tar)?; - let mut builder = Builder::new(output); + // Prepare the .tar.gz file + let output = fs::File::create(&tar_gz)?; + let gz = GzEncoder::new(output, flate2::Compression::Best); + + // Prepare the .tar.xz file + let output = fs::File::create(&tar_xz)?; + let xz = XzEncoder::new(output, 9); + + // Write the tar into both encoded files + let mut builder = Builder::new(Tee(gz, xz)); for path in paths { let path = Path::new(&path); let src = Path::new(&self.work_dir).join(path); @@ -66,25 +71,11 @@ impl Tarballer { builder.append_file(path, &mut src)?; } } - let mut input = builder.into_inner()?; - - // Write the .tar.xz file - let output = fs::File::create(&tar_xz)?; - let mut encoded = XzEncoder::new(output, 9); - input.seek(SeekFrom::Start(0))?; - io::copy(&mut input, &mut encoded)?; - encoded.finish()?; - - // Write the .tar.gz file - let output = fs::File::create(&tar_gz)?; - let mut encoded = GzEncoder::new(output, flate2::Compression::Best); - input.seek(SeekFrom::Start(0))?; - io::copy(&mut input, &mut encoded)?; - encoded.finish()?; + let Tee(gz, xz) = builder.into_inner()?; - // Remove the .tar file - drop(input); - fs::remove_file(&tar)?; + // Finish both encoded files + gz.finish()?; + xz.finish()?; Ok(()) } @@ -109,3 +100,17 @@ fn get_recursive_paths(root: &Path, name: &Path) -> io::Result> { } Ok(paths) } + +struct Tee(A, B); + +impl Write for Tee { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write_all(buf) + .and(self.1.write_all(buf)) + .and(Ok(buf.len())) + } + + fn flush(&mut self) -> io::Result<()> { + self.0.flush().and(self.1.flush()) + } +} From 890926efebaf89b5062f60302626e0d28b050e44 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 09:31:49 -0700 Subject: [PATCH 15/21] Write all directories to the tarball for rustup.rs#1092 --- src/tarballer.rs | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index eaa4573..799d7e5 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -48,8 +48,9 @@ impl Tarballer { // Sort files by their suffix, to group files with the same name from // different locations (likely identical) and files with the same // extension (likely containing similar data). - let mut paths = get_recursive_paths(self.work_dir.as_ref(), self.input.as_ref())?; - paths.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); + let (dirs, mut files) = get_recursive_paths(self.work_dir.as_ref(), + self.input.as_ref())?; + files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); // Prepare the .tar.gz file let output = fs::File::create(&tar_gz)?; @@ -59,17 +60,17 @@ impl Tarballer { let output = fs::File::create(&tar_xz)?; let xz = XzEncoder::new(output, 9); - // Write the tar into both encoded files + // Write the tar into both encoded files. We write all directories + // first, so files may be directly created. (see rustup.rs#1092) let mut builder = Builder::new(Tee(gz, xz)); - for path in paths { - let path = Path::new(&path); - let src = Path::new(&self.work_dir).join(path); - if path.is_dir() { - builder.append_dir(path, src)?; - } else { - let mut src = fs::File::open(src)?; - builder.append_file(path, &mut src)?; - } + for path in dirs { + let src = Path::new(&self.work_dir).join(&path); + builder.append_dir(&path, src)?; + } + for path in files { + let src = Path::new(&self.work_dir).join(&path); + fs::File::open(src) + .and_then(|mut file| builder.append_file(&path, &mut file))?; } let Tee(gz, xz) = builder.into_inner()?; @@ -81,24 +82,22 @@ impl Tarballer { } } -fn get_recursive_paths(root: &Path, name: &Path) -> io::Result> { - let mut paths = vec![]; +/// Returns all `(directories, files)` under the source path +fn get_recursive_paths(root: &Path, name: &Path) -> io::Result<(Vec, Vec)> { + let mut dirs = vec![]; + let mut files = vec![]; for entry in WalkDir::new(root.join(name)).min_depth(1) { let entry = entry?; let path = entry.path().strip_prefix(root).unwrap(); let path = path.to_str().unwrap().to_owned(); if entry.file_type().is_dir() { - // Include only empty dirs, as others get add via their contents. - // FIXME: do we really need empty dirs at all? - if fs::read_dir(entry.path())?.next().is_none() { - paths.push(path); - } + dirs.push(path); } else { - paths.push(path); + files.push(path); } } - Ok(paths) + Ok((dirs, files)) } struct Tee(A, B); From 678aa1158d7215d62b54ce50166d66938a76cd2a Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 11:07:50 -0700 Subject: [PATCH 16/21] Deal with remove_dir_all complications --- Cargo.toml | 5 + src/combiner.rs | 3 +- src/generator.rs | 3 +- src/lib.rs | 11 + src/remove_dir_all.rs | 835 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 855 insertions(+), 2 deletions(-) create mode 100644 src/remove_dir_all.rs diff --git a/Cargo.toml b/Cargo.toml index c3d8f9e..5d44cd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,8 @@ xz2 = "0.1.3" [dependencies.clap] features = ["yaml"] version = "2.22.1" + +[target."cfg(windows)".dependencies] +lazy_static = "0.2.8" +kernel32-sys = "0.2.2" +winapi = "0.2.8" diff --git a/src/combiner.rs b/src/combiner.rs index 81e7c47..4e52b24 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -16,6 +16,7 @@ use tar::Archive; use super::Scripter; use super::Tarballer; +use remove_dir_all::*; use util::*; actor!{ @@ -57,7 +58,7 @@ impl Combiner { let package_dir = Path::new(&self.work_dir).join(&self.package_name); if package_dir.exists() { - fs::remove_dir_all(&package_dir)?; + remove_dir_all(&package_dir)?; } fs::create_dir_all(&package_dir)?; diff --git a/src/generator.rs b/src/generator.rs index 12462a7..5c688d0 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -14,6 +14,7 @@ use std::path::Path; use super::Scripter; use super::Tarballer; +use remove_dir_all::*; use util::*; actor!{ @@ -61,7 +62,7 @@ impl Generator { let package_dir = Path::new(&self.work_dir).join(&self.package_name); if package_dir.exists() { - fs::remove_dir_all(&package_dir)?; + remove_dir_all(&package_dir)?; } // Copy the image and write the manifest diff --git a/src/lib.rs b/src/lib.rs index 885461c..0b0b42b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,20 @@ extern crate tar; extern crate walkdir; extern crate xz2; +#[cfg(windows)] +extern crate winapi; +#[cfg(windows)] +extern crate kernel32; +#[cfg(windows)] +#[macro_use] +extern crate lazy_static; + #[macro_use] mod util; +// deal with OS complications (cribbed from rustup.rs) +mod remove_dir_all; + mod combiner; mod generator; mod scripter; diff --git a/src/remove_dir_all.rs b/src/remove_dir_all.rs new file mode 100644 index 0000000..778b3a1 --- /dev/null +++ b/src/remove_dir_all.rs @@ -0,0 +1,835 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![allow(non_snake_case)] + +use std::path::Path; +use std::io; + +#[cfg(not(windows))] +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + ::std::fs::remove_dir_all(path) +} + +#[cfg(windows)] +pub fn remove_dir_all(path: &Path) -> io::Result<()> { + win::remove_dir_all(path) +} + +#[cfg(windows)] +mod win { + use winapi::{ + FileBasicInfo, + FILE_BASIC_INFO, + FALSE, + FileRenameInfo, + FILE_RENAME_INFO, + c_ushort, + c_uint, + FILETIME, + FILE_ATTRIBUTE_READONLY, + FILE_ATTRIBUTE_REPARSE_POINT, + FILE_ATTRIBUTE_DIRECTORY, + WIN32_FIND_DATAW, + ERROR_NO_MORE_FILES, + OPEN_EXISTING, + OPEN_ALWAYS, + TRUNCATE_EXISTING, + CREATE_ALWAYS, + CREATE_NEW, + GENERIC_READ, + GENERIC_WRITE, + FILE_GENERIC_WRITE, + FILE_WRITE_DATA, + FILE_SHARE_READ, + FILE_SHARE_WRITE, + FILE_SHARE_DELETE, + FILE_FLAG_DELETE_ON_CLOSE, + DELETE, + FILE_WRITE_ATTRIBUTES, + FILE_INFO_BY_HANDLE_CLASS, + HANDLE, + ERROR_INSUFFICIENT_BUFFER, + FILE_READ_ATTRIBUTES, + FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, + ERROR_CALL_NOT_IMPLEMENTED, + DWORD, + BOOL, + LPVOID, + INVALID_HANDLE_VALUE, + LPCWSTR, + SECURITY_SQOS_PRESENT, + FSCTL_GET_REPARSE_POINT, + BY_HANDLE_FILE_INFORMATION, + IO_REPARSE_TAG_SYMLINK, + IO_REPARSE_TAG_MOUNT_POINT, + }; + + use kernel32::{ + CreateFileW, + GetFileInformationByHandle, + CloseHandle, + GetLastError, + SetLastError, + DeviceIoControl, + GetModuleHandleW, + GetProcAddress, + FindNextFileW, + FindFirstFileW, + }; + + use std::ptr; + use std::sync::Arc; + use std::path::{PathBuf, Path}; + use std::mem; + use std::io; + use std::ffi::{OsStr, OsString}; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + + pub fn remove_dir_all(path: &Path) -> io::Result<()> { + // On Windows it is not enough to just recursively remove the contents of a + // directory and then the directory itself. Deleting does not happen + // instantaneously, but is scheduled. + // To work around this, we move the file or directory to some `base_dir` + // right before deletion to avoid races. + // + // As `base_dir` we choose the parent dir of the directory we want to + // remove. We very probably have permission to create files here, as we + // already need write permission in this dir to delete the directory. And it + // should be on the same volume. + // + // To handle files with names like `CON` and `morse .. .`, and when a + // directory structure is so deep it needs long path names the path is first + // converted to a `//?/`-path with `get_path()`. + // + // To make sure we don't leave a moved file laying around if the process + // crashes before we can delete the file, we do all operations on an file + // handle. By opening a file with `FILE_FLAG_DELETE_ON_CLOSE` Windows will + // always delete the file when the handle closes. + // + // All files are renamed to be in the `base_dir`, and have their name + // changed to "rm-". After every rename the counter is increased. + // Rename should not overwrite possibly existing files in the base dir. So + // if it fails with `AlreadyExists`, we just increase the counter and try + // again. + // + // For read-only files and directories we first have to remove the read-only + // attribute before we can move or delete them. This also removes the + // attribute from possible hardlinks to the file, so just before closing we + // restore the read-only attribute. + // + // If 'path' points to a directory symlink or junction we should not + // recursively remove the target of the link, but only the link itself. + // + // Moving and deleting is guaranteed to succeed if we are able to open the + // file with `DELETE` permission. If others have the file open we only have + // `DELETE` permission if they have specified `FILE_SHARE_DELETE`. We can + // also delete the file now, but it will not disappear until all others have + // closed the file. But no-one can open the file after we have flagged it + // for deletion. + + // Open the path once to get the canonical path, file type and attributes. + let (path, metadata) = { + let mut opts = OpenOptions::new(); + opts.access_mode(FILE_READ_ATTRIBUTES); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OPEN_REPARSE_POINT); + let file = try!(File::open(path, &opts)); + (try!(get_path(&file)), try!(file.file_attr())) + }; + + let mut ctx = RmdirContext { + base_dir: match path.parent() { + Some(dir) => dir, + None => return Err(io::Error::new(io::ErrorKind::PermissionDenied, + "can't delete root directory")) + }, + readonly: metadata.perm().readonly(), + counter: 0, + }; + + let filetype = metadata.file_type(); + if filetype.is_dir() { + remove_dir_all_recursive(path.as_ref(), &mut ctx) + } else if filetype.is_symlink_dir() { + remove_item(path.as_ref(), &mut ctx) + } else { + Err(io::Error::new(io::ErrorKind::PermissionDenied, "Not a directory")) + } + } + + + fn readdir(p: &Path) -> io::Result { + let root = p.to_path_buf(); + let star = p.join("*"); + let path = try!(to_u16s(&star)); + + unsafe { + let mut wfd = mem::zeroed(); + let find_handle = FindFirstFileW(path.as_ptr(), &mut wfd); + if find_handle != INVALID_HANDLE_VALUE { + Ok(ReadDir { + handle: FindNextFileHandle(find_handle), + root: Arc::new(root), + first: Some(wfd), + }) + } else { + Err(io::Error::last_os_error()) + } + } + } + + struct RmdirContext<'a> { + base_dir: &'a Path, + readonly: bool, + counter: u64, + } + + fn remove_dir_all_recursive(path: &Path, ctx: &mut RmdirContext) + -> io::Result<()> { + let dir_readonly = ctx.readonly; + for child in try!(readdir(path)) { + let child = try!(child); + let child_type = try!(child.file_type()); + ctx.readonly = try!(child.metadata()).perm().readonly(); + if child_type.is_dir() { + try!(remove_dir_all_recursive(&child.path(), ctx)); + } else { + try!(remove_item(&child.path().as_ref(), ctx)); + } + } + ctx.readonly = dir_readonly; + remove_item(path, ctx) + } + + fn remove_item(path: &Path, ctx: &mut RmdirContext) -> io::Result<()> { + if !ctx.readonly { + let mut opts = OpenOptions::new(); + opts.access_mode(DELETE); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | // delete directory + FILE_FLAG_OPEN_REPARSE_POINT | // delete symlink + FILE_FLAG_DELETE_ON_CLOSE); + let file = try!(File::open(path, &opts)); + move_item(&file, ctx) + } else { + // remove read-only permision + try!(set_perm(&path, FilePermissions::new())); + // move and delete file, similar to !readonly. + // only the access mode is different. + let mut opts = OpenOptions::new(); + opts.access_mode(DELETE | FILE_WRITE_ATTRIBUTES); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OPEN_REPARSE_POINT | + FILE_FLAG_DELETE_ON_CLOSE); + let file = try!(File::open(path, &opts)); + try!(move_item(&file, ctx)); + // restore read-only flag just in case there are other hard links + let mut perm = FilePermissions::new(); + perm.set_readonly(true); + let _ = file.set_perm(perm); // ignore if this fails + Ok(()) + } + } + + macro_rules! compat_fn { + ($module:ident: $( + fn $symbol:ident($($argname:ident: $argtype:ty),*) + -> $rettype:ty { + $($body:expr);* + } + )*) => ($( + #[allow(unused_variables)] + unsafe fn $symbol($($argname: $argtype),*) -> $rettype { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::mem; + use std::ffi::CString; + type F = unsafe extern "system" fn($($argtype),*) -> $rettype; + + lazy_static! { static ref PTR: AtomicUsize = AtomicUsize::new(0);} + + fn lookup(module: &str, symbol: &str) -> Option { + let mut module: Vec = module.encode_utf16().collect(); + module.push(0); + let symbol = CString::new(symbol).unwrap(); + unsafe { + let handle = GetModuleHandleW(module.as_ptr()); + match GetProcAddress(handle, symbol.as_ptr()) as usize { + 0 => None, + n => Some(n), + } + } + } + + fn store_func(ptr: &AtomicUsize, module: &str, symbol: &str, + fallback: usize) -> usize { + let value = lookup(module, symbol).unwrap_or(fallback); + ptr.store(value, Ordering::SeqCst); + value + } + + fn load() -> usize { + store_func(&PTR, stringify!($module), stringify!($symbol), fallback as usize) + } + unsafe extern "system" fn fallback($($argname: $argtype),*) + -> $rettype { + $($body);* + } + + let addr = match PTR.load(Ordering::SeqCst) { + 0 => load(), + n => n, + }; + mem::transmute::(addr)($($argname),*) + } + )*) + } + + compat_fn! { + kernel32: + fn GetFinalPathNameByHandleW(_hFile: HANDLE, + _lpszFilePath: LPCWSTR, + _cchFilePath: DWORD, + _dwFlags: DWORD) -> DWORD { + SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 + } + fn SetFileInformationByHandle(_hFile: HANDLE, + _FileInformationClass: FILE_INFO_BY_HANDLE_CLASS, + _lpFileInformation: LPVOID, + _dwBufferSize: DWORD) -> BOOL { + SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0 + } + } + + fn cvt(i: i32) -> io::Result { + if i == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(i) + } + } + + fn to_u16s>(s: S) -> io::Result> { + fn inner(s: &OsStr) -> io::Result> { + let mut maybe_result: Vec = s.encode_wide().collect(); + if maybe_result.iter().any(|&u| u == 0) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, + "strings passed to WinAPI cannot contain NULs")); + } + maybe_result.push(0); + Ok(maybe_result) + } + inner(s.as_ref()) + } + + fn truncate_utf16_at_nul<'a>(v: &'a [u16]) -> &'a [u16] { + match v.iter().position(|c| *c == 0) { + // don't include the 0 + Some(i) => &v[..i], + None => v + } + } + + fn fill_utf16_buf(mut f1: F1, f2: F2) -> io::Result + where F1: FnMut(*mut u16, DWORD) -> DWORD, + F2: FnOnce(&[u16]) -> T + { + // Start off with a stack buf but then spill over to the heap if we end up + // needing more space. + let mut stack_buf = [0u16; 512]; + let mut heap_buf = Vec::new(); + unsafe { + let mut n = stack_buf.len(); + loop { + let buf = if n <= stack_buf.len() { + &mut stack_buf[..] + } else { + let extra = n - heap_buf.len(); + heap_buf.reserve(extra); + heap_buf.set_len(n); + &mut heap_buf[..] + }; + + // This function is typically called on windows API functions which + // will return the correct length of the string, but these functions + // also return the `0` on error. In some cases, however, the + // returned "correct length" may actually be 0! + // + // To handle this case we call `SetLastError` to reset it to 0 and + // then check it again if we get the "0 error value". If the "last + // error" is still 0 then we interpret it as a 0 length buffer and + // not an actual error. + SetLastError(0); + let k = match f1(buf.as_mut_ptr(), n as DWORD) { + 0 if GetLastError() == 0 => 0, + 0 => return Err(io::Error::last_os_error()), + n => n, + } as usize; + if k == n && GetLastError() == ERROR_INSUFFICIENT_BUFFER { + n *= 2; + } else if k >= n { + n = k; + } else { + return Ok(f2(&buf[..k])) + } + } + } + } + + #[derive(Clone, PartialEq, Eq, Debug, Default)] + struct FilePermissions { readonly: bool } + + impl FilePermissions { + fn new() -> FilePermissions { Default::default() } + fn readonly(&self) -> bool { self.readonly } + fn set_readonly(&mut self, readonly: bool) { self.readonly = readonly } + } + + #[derive(Clone)] + struct OpenOptions { + // generic + read: bool, + write: bool, + append: bool, + truncate: bool, + create: bool, + create_new: bool, + // system-specific + custom_flags: u32, + access_mode: Option, + attributes: DWORD, + share_mode: DWORD, + security_qos_flags: DWORD, + security_attributes: usize, // FIXME: should be a reference + } + + impl OpenOptions { + fn new() -> OpenOptions { + OpenOptions { + // generic + read: false, + write: false, + append: false, + truncate: false, + create: false, + create_new: false, + // system-specific + custom_flags: 0, + access_mode: None, + share_mode: FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + attributes: 0, + security_qos_flags: 0, + security_attributes: 0, + } + } + fn custom_flags(&mut self, flags: u32) { self.custom_flags = flags; } + fn access_mode(&mut self, access_mode: u32) { self.access_mode = Some(access_mode); } + + fn get_access_mode(&self) -> io::Result { + const ERROR_INVALID_PARAMETER: i32 = 87; + + match (self.read, self.write, self.append, self.access_mode) { + (_, _, _, Some(mode)) => Ok(mode), + (true, false, false, None) => Ok(GENERIC_READ), + (false, true, false, None) => Ok(GENERIC_WRITE), + (true, true, false, None) => Ok(GENERIC_READ | GENERIC_WRITE), + (false, _, true, None) => Ok(FILE_GENERIC_WRITE & !FILE_WRITE_DATA), + (true, _, true, None) => Ok(GENERIC_READ | + (FILE_GENERIC_WRITE & !FILE_WRITE_DATA)), + (false, false, false, None) => Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)), + } + } + + fn get_creation_mode(&self) -> io::Result { + const ERROR_INVALID_PARAMETER: i32 = 87; + + match (self.write, self.append) { + (true, false) => {} + (false, false) => + if self.truncate || self.create || self.create_new { + return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)); + }, + (_, true) => + if self.truncate && !self.create_new { + return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)); + }, + } + + Ok(match (self.create, self.truncate, self.create_new) { + (false, false, false) => OPEN_EXISTING, + (true, false, false) => OPEN_ALWAYS, + (false, true, false) => TRUNCATE_EXISTING, + (true, true, false) => CREATE_ALWAYS, + (_, _, true) => CREATE_NEW, + }) + } + + fn get_flags_and_attributes(&self) -> DWORD { + self.custom_flags | + self.attributes | + self.security_qos_flags | + if self.security_qos_flags != 0 { SECURITY_SQOS_PRESENT } else { 0 } | + if self.create_new { FILE_FLAG_OPEN_REPARSE_POINT } else { 0 } + } + } + + struct File { handle: Handle } + + impl File { + fn open(path: &Path, opts: &OpenOptions) -> io::Result { + let path = try!(to_u16s(path)); + let handle = unsafe { + CreateFileW(path.as_ptr(), + try!(opts.get_access_mode()), + opts.share_mode, + opts.security_attributes as *mut _, + try!(opts.get_creation_mode()), + opts.get_flags_and_attributes(), + ptr::null_mut()) + }; + if handle == INVALID_HANDLE_VALUE { + Err(io::Error::last_os_error()) + } else { + Ok(File { handle: Handle::new(handle) }) + } + } + + fn file_attr(&self) -> io::Result { + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed(); + try!(cvt(GetFileInformationByHandle(self.handle.raw(), + &mut info))); + let mut attr = FileAttr { + attributes: info.dwFileAttributes, + creation_time: info.ftCreationTime, + last_access_time: info.ftLastAccessTime, + last_write_time: info.ftLastWriteTime, + file_size: ((info.nFileSizeHigh as u64) << 32) | (info.nFileSizeLow as u64), + reparse_tag: 0, + }; + if attr.is_reparse_point() { + let mut b = [0; MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + if let Ok((_, buf)) = self.reparse_point(&mut b) { + attr.reparse_tag = buf.ReparseTag; + } + } + Ok(attr) + } + } + + fn set_attributes(&self, attr: DWORD) -> io::Result<()> { + let mut info = FILE_BASIC_INFO { + CreationTime: 0, // do not change + LastAccessTime: 0, // do not change + LastWriteTime: 0, // do not change + ChangeTime: 0, // do not change + FileAttributes: attr, + }; + let size = mem::size_of_val(&info); + try!(cvt(unsafe { + SetFileInformationByHandle(self.handle.raw(), + FileBasicInfo, + &mut info as *mut _ as *mut _, + size as DWORD) + })); + Ok(()) + } + + fn rename(&self, new: &Path, replace: bool) -> io::Result<()> { + // &self must be opened with DELETE permission + use std::iter; + #[cfg(target_arch = "x86")] + const STRUCT_SIZE: usize = 12; + #[cfg(target_arch = "x86_64")] + const STRUCT_SIZE: usize = 20; + + // FIXME: check for internal NULs in 'new' + let mut data: Vec = iter::repeat(0u16).take(STRUCT_SIZE/2) + .chain(new.as_os_str().encode_wide()) + .collect(); + data.push(0); + let size = data.len() * 2; + + unsafe { + // Thanks to alignment guarantees on Windows this works + // (8 for 32-bit and 16 for 64-bit) + let mut info = data.as_mut_ptr() as *mut FILE_RENAME_INFO; + // The type of ReplaceIfExists is BOOL, but it actually expects a + // BOOLEAN. This means true is -1, not c::TRUE. + (*info).ReplaceIfExists = if replace { -1 } else { FALSE }; + (*info).RootDirectory = ptr::null_mut(); + (*info).FileNameLength = (size - STRUCT_SIZE) as DWORD; + try!(cvt(SetFileInformationByHandle(self.handle().raw(), + FileRenameInfo, + data.as_mut_ptr() as *mut _ as *mut _, + size as DWORD))); + Ok(()) + } + } + fn set_perm(&self, perm: FilePermissions) -> io::Result<()> { + let attr = try!(self.file_attr()).attributes; + if perm.readonly == (attr & FILE_ATTRIBUTE_READONLY != 0) { + Ok(()) + } else if perm.readonly { + self.set_attributes(attr | FILE_ATTRIBUTE_READONLY) + } else { + self.set_attributes(attr & !FILE_ATTRIBUTE_READONLY) + } + } + + fn handle(&self) -> &Handle { &self.handle } + + fn reparse_point<'a>(&self, + space: &'a mut [u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE]) + -> io::Result<(DWORD, &'a REPARSE_DATA_BUFFER)> { + unsafe { + let mut bytes = 0; + try!(cvt({ + DeviceIoControl(self.handle.raw(), + FSCTL_GET_REPARSE_POINT, + ptr::null_mut(), + 0, + space.as_mut_ptr() as *mut _, + space.len() as DWORD, + &mut bytes, + ptr::null_mut()) + })); + Ok((bytes, &*(space.as_ptr() as *const REPARSE_DATA_BUFFER))) + } + } + } + + + #[derive(Copy, Clone, PartialEq, Eq, Hash)] + enum FileType { + Dir, File, SymlinkFile, SymlinkDir, ReparsePoint, MountPoint, + } + + impl FileType { + fn new(attrs: DWORD, reparse_tag: DWORD) -> FileType { + match (attrs & FILE_ATTRIBUTE_DIRECTORY != 0, + attrs & FILE_ATTRIBUTE_REPARSE_POINT != 0, + reparse_tag) { + (false, false, _) => FileType::File, + (true, false, _) => FileType::Dir, + (false, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkFile, + (true, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkDir, + (true, true, IO_REPARSE_TAG_MOUNT_POINT) => FileType::MountPoint, + (_, true, _) => FileType::ReparsePoint, + // Note: if a _file_ has a reparse tag of the type IO_REPARSE_TAG_MOUNT_POINT it is + // invalid, as junctions always have to be dirs. We set the filetype to ReparsePoint + // to indicate it is something symlink-like, but not something you can follow. + } + } + + fn is_dir(&self) -> bool { *self == FileType::Dir } + fn is_symlink_dir(&self) -> bool { + *self == FileType::SymlinkDir || *self == FileType::MountPoint + } + } + + impl DirEntry { + fn new(root: &Arc, wfd: &WIN32_FIND_DATAW) -> Option { + let first_bytes = &wfd.cFileName[0..3]; + if first_bytes.starts_with(&[46, 0]) || first_bytes.starts_with(&[46, 46, 0]) { + None + } else { + Some(DirEntry { + root: root.clone(), + data: *wfd, + }) + } + } + + fn path(&self) -> PathBuf { + self.root.join(&self.file_name()) + } + + fn file_name(&self) -> OsString { + let filename = truncate_utf16_at_nul(&self.data.cFileName); + OsString::from_wide(filename) + } + + fn file_type(&self) -> io::Result { + Ok(FileType::new(self.data.dwFileAttributes, + /* reparse_tag = */ self.data.dwReserved0)) + } + + fn metadata(&self) -> io::Result { + Ok(FileAttr { + attributes: self.data.dwFileAttributes, + creation_time: self.data.ftCreationTime, + last_access_time: self.data.ftLastAccessTime, + last_write_time: self.data.ftLastWriteTime, + file_size: ((self.data.nFileSizeHigh as u64) << 32) | (self.data.nFileSizeLow as u64), + reparse_tag: if self.data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + // reserved unless this is a reparse point + self.data.dwReserved0 + } else { + 0 + }, + }) + } + } + + + + struct DirEntry { + root: Arc, + data: WIN32_FIND_DATAW, + } + + struct ReadDir { + handle: FindNextFileHandle, + root: Arc, + first: Option, + } + + impl Iterator for ReadDir { + type Item = io::Result; + fn next(&mut self) -> Option> { + if let Some(first) = self.first.take() { + if let Some(e) = DirEntry::new(&self.root, &first) { + return Some(Ok(e)); + } + } + unsafe { + let mut wfd = mem::zeroed(); + loop { + if FindNextFileW(self.handle.0, &mut wfd) == 0 { + if GetLastError() == ERROR_NO_MORE_FILES { + return None + } else { + return Some(Err(io::Error::last_os_error())) + } + } + if let Some(e) = DirEntry::new(&self.root, &wfd) { + return Some(Ok(e)) + } + } + } + } + } + + + #[derive(Clone)] + struct FileAttr { + attributes: DWORD, + creation_time: FILETIME, + last_access_time: FILETIME, + last_write_time: FILETIME, + file_size: u64, + reparse_tag: DWORD, + } + + impl FileAttr { + fn perm(&self) -> FilePermissions { + FilePermissions { + readonly: self.attributes & FILE_ATTRIBUTE_READONLY != 0 + } + } + + fn file_type(&self) -> FileType { + FileType::new(self.attributes, self.reparse_tag) + } + + fn is_reparse_point(&self) -> bool { + self.attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + } + } + + #[repr(C)] + struct REPARSE_DATA_BUFFER { + ReparseTag: c_uint, + ReparseDataLength: c_ushort, + Reserved: c_ushort, + rest: (), + } + + const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024; + + + /// An owned container for `HANDLE` object, closing them on Drop. + /// + /// All methods are inherited through a `Deref` impl to `RawHandle` + struct Handle(RawHandle); + + use std::ops::Deref; + + /// A wrapper type for `HANDLE` objects to give them proper Send/Sync inference + /// as well as Rust-y methods. + /// + /// This does **not** drop the handle when it goes out of scope, use `Handle` + /// instead for that. + #[derive(Copy, Clone)] + struct RawHandle(HANDLE); + + unsafe impl Send for RawHandle {} + unsafe impl Sync for RawHandle {} + + impl Handle { + fn new(handle: HANDLE) -> Handle { + Handle(RawHandle::new(handle)) + } + } + + impl Deref for Handle { + type Target = RawHandle; + fn deref(&self) -> &RawHandle { &self.0 } + } + + impl Drop for Handle { + fn drop(&mut self) { + unsafe { let _ = CloseHandle(self.raw()); } + } + } + + impl RawHandle { + fn new(handle: HANDLE) -> RawHandle { + RawHandle(handle) + } + + fn raw(&self) -> HANDLE { self.0 } + } + + struct FindNextFileHandle(HANDLE); + + fn get_path(f: &File) -> io::Result { + fill_utf16_buf(|buf, sz| unsafe { + GetFinalPathNameByHandleW(f.handle.raw(), buf, sz, + VOLUME_NAME_DOS) + }, |buf| { + PathBuf::from(OsString::from_wide(buf)) + }) + } + + fn move_item(file: &File, ctx: &mut RmdirContext) -> io::Result<()> { + let mut tmpname = ctx.base_dir.join(format!{"rm-{}", ctx.counter}); + ctx.counter += 1; + // Try to rename the file. If it already exists, just retry with an other + // filename. + while let Err(err) = file.rename(tmpname.as_ref(), false) { + if err.kind() != io::ErrorKind::AlreadyExists { return Err(err) }; + tmpname = ctx.base_dir.join(format!("rm-{}", ctx.counter)); + ctx.counter += 1; + } + Ok(()) + } + + fn set_perm(path: &Path, perm: FilePermissions) -> io::Result<()> { + let mut opts = OpenOptions::new(); + opts.access_mode(FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES); + opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS); + let file = try!(File::open(path, &opts)); + file.set_perm(perm) + } + + const VOLUME_NAME_DOS: DWORD = 0x0; +} From 4f6e020b179ef79a3e3717f80132ad1a4875a669 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 14:14:05 -0700 Subject: [PATCH 17/21] Move the combined components instead of copying --- src/combiner.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/combiner.rs b/src/combiner.rs index 4e52b24..3bce4a1 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -83,15 +83,14 @@ impl Combiner { return Err(io::Error::new(io::ErrorKind::Other, msg)); } - // Copy components to new combined installer + // Move components to the new combined installer let mut pkg_components = String::new(); fs::File::open(pkg_dir.join("components"))? .read_to_string(&mut pkg_components)?; for component in pkg_components.split_whitespace() { - // All we need to do is copy the component directory + // All we need to do is move the component directory let component_dir = package_dir.join(&component); - fs::create_dir(&component_dir)?; - copy_recursive(&pkg_dir.join(&component), &component_dir)?; + fs::rename(&pkg_dir.join(&component), &component_dir)?; // Merge the component name writeln!(&components, "{}", component)?; From d3d56be1697b5a75a618da631be4b0418d8fae93 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 15:14:33 -0700 Subject: [PATCH 18/21] Use error-chain for more informative errors --- Cargo.toml | 1 + src/combiner.rs | 51 +++++++++++++++++++++------------------- src/generator.rs | 30 ++++++++++++------------ src/lib.rs | 13 +++++++++++ src/main.rs | 52 ++++++++++++++++++++--------------------- src/scripter.rs | 11 +++++---- src/tarballer.rs | 53 +++++++++++++++++++++++++++--------------- src/util.rs | 60 ++++++++++++++++++++++++++++++++++++++++++------ 8 files changed, 178 insertions(+), 93 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5d44cd2..f998caa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ name = "rust-installer" path = "src/main.rs" [dependencies] +error-chain = "0.10.0" flate2 = "0.2.19" tar = "0.4.11" walkdir = "1.0.7" diff --git a/src/combiner.rs b/src/combiner.rs index 3bce4a1..512d9d5 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -9,14 +9,14 @@ // except according to those terms. use std::fs; -use std::io::{self, Read, Write}; +use std::io::{Read, Write}; use std::path::Path; use flate2::read::GzDecoder; use tar::Archive; +use errors::*; use super::Scripter; use super::Tarballer; -use remove_dir_all::*; use util::*; actor!{ @@ -53,22 +53,25 @@ actor!{ impl Combiner { /// Combine the installer tarballs - pub fn run(self) -> io::Result<()> { - fs::create_dir_all(&self.work_dir)?; + pub fn run(self) -> Result<()> { + create_dir_all(&self.work_dir)?; let package_dir = Path::new(&self.work_dir).join(&self.package_name); if package_dir.exists() { remove_dir_all(&package_dir)?; } - fs::create_dir_all(&package_dir)?; + create_dir_all(&package_dir)?; // Merge each installer into the work directory of the new installer - let components = fs::File::create(package_dir.join("components"))?; + let components = fs::File::create(package_dir.join("components")) + .chain_err(|| "failed to create a new components file")?; for input_tarball in self.input_tarballs.split(',').map(str::trim).filter(|s| !s.is_empty()) { // Extract the input tarballs - let input = fs::File::open(&input_tarball)?; - let deflated = GzDecoder::new(input)?; - Archive::new(deflated).unpack(&self.work_dir)?; + fs::File::open(&input_tarball) + .and_then(GzDecoder::new) + .and_then(|tar| Archive::new(tar).unpack(&self.work_dir)) + .chain_err(|| format!("unable to extract '{}' into '{}'", + &input_tarball, self.work_dir))?; let pkg_name = input_tarball.trim_right_matches(".tar.gz"); let pkg_name = Path::new(pkg_name).file_name().unwrap(); @@ -76,32 +79,34 @@ impl Combiner { // Verify the version number let mut version = String::new(); - fs::File::open(pkg_dir.join("rust-installer-version"))? - .read_to_string(&mut version)?; + fs::File::open(pkg_dir.join("rust-installer-version")) + .and_then(|mut file| file.read_to_string(&mut version)) + .chain_err(|| format!("failed to read version in '{}'", input_tarball))?; if version.trim().parse() != Ok(::RUST_INSTALLER_VERSION) { - let msg = format!("incorrect installer version in {}", input_tarball); - return Err(io::Error::new(io::ErrorKind::Other, msg)); + bail!("incorrect installer version in {}", input_tarball); } // Move components to the new combined installer let mut pkg_components = String::new(); - fs::File::open(pkg_dir.join("components"))? - .read_to_string(&mut pkg_components)?; + fs::File::open(pkg_dir.join("components")) + .and_then(|mut file| file.read_to_string(&mut pkg_components)) + .chain_err(|| format!("failed to read components in '{}'", input_tarball))?; for component in pkg_components.split_whitespace() { // All we need to do is move the component directory let component_dir = package_dir.join(&component); - fs::rename(&pkg_dir.join(&component), &component_dir)?; + rename(&pkg_dir.join(&component), component_dir)?; // Merge the component name - writeln!(&components, "{}", component)?; + writeln!(&components, "{}", component) + .chain_err(|| "failed to write new components")?; } } drop(components); // Write the installer version - let version = fs::File::create(package_dir.join("rust-installer-version"))?; - writeln!(&version, "{}", ::RUST_INSTALLER_VERSION)?; - drop(version); + fs::File::create(package_dir.join("rust-installer-version")) + .and_then(|file| writeln!(&file, "{}", ::RUST_INSTALLER_VERSION)) + .chain_err(|| "failed to write new installer version")?; // Copy the overlay if !self.non_installed_overlay.is_empty() { @@ -115,16 +120,16 @@ impl Combiner { .rel_manifest_dir(self.rel_manifest_dir) .success_message(self.success_message) .legacy_manifest_dirs(self.legacy_manifest_dirs) - .output_script(output_script.to_str().unwrap()); + .output_script(path_to_str(&output_script)?); scripter.run()?; // Make the tarballs - fs::create_dir_all(&self.output_dir)?; + create_dir_all(&self.output_dir)?; let output = Path::new(&self.output_dir).join(&self.package_name); let mut tarballer = Tarballer::default(); tarballer.work_dir(self.work_dir) .input(self.package_name) - .output(output.to_str().unwrap()); + .output(path_to_str(&output)?); tarballer.run()?; Ok(()) diff --git a/src/generator.rs b/src/generator.rs index 5c688d0..e76e809 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -9,12 +9,12 @@ // except according to those terms. use std::fs; -use std::io::{self, Write}; +use std::io::Write; use std::path::Path; +use errors::*; use super::Scripter; use super::Tarballer; -use remove_dir_all::*; use util::*; actor!{ @@ -57,8 +57,8 @@ actor!{ impl Generator { /// Generate the actual installer tarball - pub fn run(self) -> io::Result<()> { - fs::create_dir_all(&self.work_dir)?; + pub fn run(self) -> Result<()> { + create_dir_all(&self.work_dir)?; let package_dir = Path::new(&self.work_dir).join(&self.package_name); if package_dir.exists() { @@ -67,18 +67,18 @@ impl Generator { // Copy the image and write the manifest let component_dir = package_dir.join(&self.component_name); - fs::create_dir_all(&component_dir)?; + create_dir_all(&component_dir)?; copy_and_manifest(self.image_dir.as_ref(), &component_dir, &self.bulk_dirs)?; // Write the component name - let components = fs::File::create(package_dir.join("components"))?; - writeln!(&components, "{}", self.component_name)?; - drop(components); + fs::File::create(package_dir.join("components")) + .and_then(|file| writeln!(&file, "{}", self.component_name)) + .chain_err(|| "failed to write the component file")?; // Write the installer version (only used by combine-installers.sh) - let version = fs::File::create(package_dir.join("rust-installer-version"))?; - writeln!(&version, "{}", ::RUST_INSTALLER_VERSION)?; - drop(version); + fs::File::create(package_dir.join("rust-installer-version")) + .and_then(|file| writeln!(&file, "{}", ::RUST_INSTALLER_VERSION)) + .chain_err(|| "failed to write the installer version")?; // Copy the overlay if !self.non_installed_overlay.is_empty() { @@ -92,16 +92,16 @@ impl Generator { .rel_manifest_dir(self.rel_manifest_dir) .success_message(self.success_message) .legacy_manifest_dirs(self.legacy_manifest_dirs) - .output_script(output_script.to_str().unwrap()); + .output_script(path_to_str(&output_script)?); scripter.run()?; // Make the tarballs - fs::create_dir_all(&self.output_dir)?; + create_dir_all(&self.output_dir)?; let output = Path::new(&self.output_dir).join(&self.package_name); let mut tarballer = Tarballer::default(); tarballer.work_dir(self.work_dir) .input(self.package_name) - .output(output.to_str().unwrap()); + .output(path_to_str(&output)?); tarballer.run()?; Ok(()) @@ -109,7 +109,7 @@ impl Generator { } /// Copies the `src` directory recursively to `dst`, writing `manifest.in` too. -fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> io::Result<()> { +fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> Result<()> { let manifest = fs::File::create(dst.join("manifest.in"))?; let bulk_dirs: Vec<_> = bulk_dirs.split(',') .filter(|s| !s.is_empty()) diff --git a/src/lib.rs b/src/lib.rs index 0b0b42b..b9375df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#[macro_use] +extern crate error_chain; extern crate flate2; extern crate tar; extern crate walkdir; @@ -21,6 +23,16 @@ extern crate kernel32; #[macro_use] extern crate lazy_static; +mod errors { + error_chain!{ + foreign_links { + Io(::std::io::Error); + StripPrefix(::std::path::StripPrefixError); + WalkDir(::walkdir::Error); + } + } +} + #[macro_use] mod util; @@ -32,6 +44,7 @@ mod generator; mod scripter; mod tarballer; +pub use errors::{Result, Error, ErrorKind}; pub use combiner::Combiner; pub use generator::Generator; pub use scripter::Scripter; diff --git a/src/main.rs b/src/main.rs index 882aa57..cabffb8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,23 @@ #[macro_use] extern crate clap; +#[macro_use] +extern crate error_chain; extern crate installer; +use errors::*; use clap::{App, ArgMatches}; -use installer::*; -fn main() { +mod errors { + error_chain!{ + links { + Installer(::installer::Error, ::installer::ErrorKind); + } + } +} + +quick_main!(run); + +fn run() -> Result<()> { let yaml = load_yaml!("main.yml"); let matches = App::from_yaml(yaml).get_matches(); @@ -29,8 +41,8 @@ macro_rules! parse( } ); -fn combine(matches: &ArgMatches) { - let combiner = parse!(matches => Combiner { +fn combine(matches: &ArgMatches) -> Result<()> { + let combiner = parse!(matches => installer::Combiner { "product-name" => product_name, "package-name" => package_name, "rel-manifest-dir" => rel_manifest_dir, @@ -42,14 +54,11 @@ fn combine(matches: &ArgMatches) { "output-dir" => output_dir, }); - if let Err(e) = combiner.run() { - println!("failed to combine installers: {}", e); - std::process::exit(1); - } + combiner.run().chain_err(|| "failed to combine installers") } -fn generate(matches: &ArgMatches) { - let generator = parse!(matches => Generator { +fn generate(matches: &ArgMatches) -> Result<()> { + let generator = parse!(matches => installer::Generator { "product-name" => product_name, "component-name" => component_name, "package-name" => package_name, @@ -63,14 +72,11 @@ fn generate(matches: &ArgMatches) { "output-dir" => output_dir, }); - if let Err(e) = generator.run() { - println!("failed to generate installer: {}", e); - std::process::exit(1); - } + generator.run().chain_err(|| "failed to generate installer") } -fn script(matches: &ArgMatches) { - let scripter = parse!(matches => Scripter { +fn script(matches: &ArgMatches) -> Result<()> { + let scripter = parse!(matches => installer::Scripter { "product-name" => product_name, "rel-manifest-dir" => rel_manifest_dir, "success-message" => success_message, @@ -78,21 +84,15 @@ fn script(matches: &ArgMatches) { "output-script" => output_script, }); - if let Err(e) = scripter.run() { - println!("failed to generate installation script: {}", e); - std::process::exit(1); - } + scripter.run().chain_err(|| "failed to generate installation script") } -fn tarball(matches: &ArgMatches) { - let tarballer = parse!(matches => Tarballer { +fn tarball(matches: &ArgMatches) -> Result<()> { + let tarballer = parse!(matches => installer::Tarballer { "input" => input, "output" => output, "work-dir" => work_dir, }); - if let Err(e) = tarballer.run() { - println!("failed to generate tarballs: {}", e); - std::process::exit(1); - } + tarballer.run().chain_err(|| "failed to generate tarballs") } diff --git a/src/scripter.rs b/src/scripter.rs index f066320..9380424 100644 --- a/src/scripter.rs +++ b/src/scripter.rs @@ -9,13 +9,15 @@ // except according to those terms. use std::fs; -use std::io::{self, Write}; +use std::io::Write; // Needed to set the script mode to executable. #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; // FIXME: what about Windows? Are default ACLs executable? +use errors::*; + const TEMPLATE: &'static str = include_str!("../install-template.sh"); @@ -41,7 +43,7 @@ actor!{ impl Scripter { /// Generate the actual installer script - pub fn run(self) -> io::Result<()> { + pub fn run(self) -> Result<()> { // Replace dashes in the success message with spaces (our arg handling botches spaces) // (TODO: still needed? kept for compatibility for now...) let product_name = self.product_name.replace('-', " "); @@ -60,8 +62,9 @@ impl Scripter { let mut options = fs::OpenOptions::new(); options.write(true).create_new(true); #[cfg(unix)] options.mode(0o755); - let output = options.open(self.output_script)?; - writeln!(&output, "{}", script) + options.open(&self.output_script) + .and_then(|mut output| output.write_all(script.as_ref())) + .chain_err(|| format!("failed to write output script '{}'", self.output_script)) } } diff --git a/src/tarballer.rs b/src/tarballer.rs index 799d7e5..9f9fbcc 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -18,6 +18,9 @@ use tar::Builder; use walkdir::WalkDir; use xz2::write::XzEncoder; +use errors::*; +use util::*; + actor!{ #[derive(Debug)] pub struct Tarballer { @@ -27,37 +30,39 @@ actor!{ /// The prefix of the tarballs output: String = "./dist", - /// The fold in which the input is to be found + /// The folder in which the input is to be found work_dir: String = "./workdir", } } impl Tarballer { /// Generate the actual tarballs - pub fn run(self) -> io::Result<()> { + pub fn run(self) -> Result<()> { let tar_gz = self.output.clone() + ".tar.gz"; let tar_xz = self.output.clone() + ".tar.xz"; // Remove any existing files for file in &[&tar_gz, &tar_xz] { if Path::new(file).exists() { - fs::remove_file(file)?; + remove_file(file)?; } } // Sort files by their suffix, to group files with the same name from // different locations (likely identical) and files with the same // extension (likely containing similar data). - let (dirs, mut files) = get_recursive_paths(self.work_dir.as_ref(), - self.input.as_ref())?; + let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input) + .chain_err(|| "failed to collect file paths")?; files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); // Prepare the .tar.gz file - let output = fs::File::create(&tar_gz)?; + let output = fs::File::create(&tar_gz) + .chain_err(|| "failed to create .tar.gz file")?; let gz = GzEncoder::new(output, flate2::Compression::Best); // Prepare the .tar.xz file - let output = fs::File::create(&tar_xz)?; + let output = fs::File::create(&tar_xz) + .chain_err(|| "failed to create .tar.xz file")?; let xz = XzEncoder::new(output, 9); // Write the tar into both encoded files. We write all directories @@ -65,36 +70,48 @@ impl Tarballer { let mut builder = Builder::new(Tee(gz, xz)); for path in dirs { let src = Path::new(&self.work_dir).join(&path); - builder.append_dir(&path, src)?; + builder.append_dir(&path, &src) + .chain_err(|| format!("failed to tar dir '{}'", src.display()))?; } for path in files { let src = Path::new(&self.work_dir).join(&path); - fs::File::open(src) - .and_then(|mut file| builder.append_file(&path, &mut file))?; + fs::File::open(&src) + .and_then(|mut file| builder.append_file(&path, &mut file)) + .chain_err(|| format!("failed to tar file '{}'", src.display()))?; } - let Tee(gz, xz) = builder.into_inner()?; + let Tee(gz, xz) = builder.into_inner() + .chain_err(|| "failed to finish writing .tar stream")?; // Finish both encoded files - gz.finish()?; - xz.finish()?; + gz.finish().chain_err(|| "failed to finish .tar.gz file")?; + xz.finish().chain_err(|| "failed to finish .tar.xz file")?; Ok(()) } } /// Returns all `(directories, files)` under the source path -fn get_recursive_paths(root: &Path, name: &Path) -> io::Result<(Vec, Vec)> { +fn get_recursive_paths(root: P, name: Q) -> Result<(Vec, Vec)> + where P: AsRef, Q: AsRef +{ + let root = root.as_ref(); + let name = name.as_ref(); + + if !name.is_relative() && !name.starts_with(root) { + bail!("input '{}' is not in work dir '{}'", name.display(), root.display()); + } + let mut dirs = vec![]; let mut files = vec![]; for entry in WalkDir::new(root.join(name)).min_depth(1) { let entry = entry?; - let path = entry.path().strip_prefix(root).unwrap(); - let path = path.to_str().unwrap().to_owned(); + let path = entry.path().strip_prefix(root)?; + let path = path_to_str(&path)?; if entry.file_type().is_dir() { - dirs.push(path); + dirs.push(path.to_owned()); } else { - files.push(path); + files.push(path.to_owned()); } } Ok((dirs, files)) diff --git a/src/util.rs b/src/util.rs index f086369..22df766 100644 --- a/src/util.rs +++ b/src/util.rs @@ -10,31 +10,77 @@ use std::fs; -use std::io; use std::path::Path; use walkdir::WalkDir; +use errors::*; + +/// Convert a `&Path` to a UTF-8 `&str` +pub fn path_to_str(path: &Path) -> Result<&str> { + path.to_str().ok_or_else(|| { + ErrorKind::Msg(format!("path is not valid UTF-8 '{}'", path.display())).into() + }) +} + +/// Wrap `fs::copy` with a nicer error message +pub fn copy, Q: AsRef>(from: P, to: Q) -> Result { + fs::copy(&from, &to) + .chain_err(|| format!("failed to copy '{}' to '{}'", + from.as_ref().display(), to.as_ref().display())) +} + +/// Wrap `fs::create_dir` with a nicer error message +pub fn create_dir>(path: P) -> Result<()> { + fs::create_dir(&path) + .chain_err(|| format!("failed to create dir '{}'", path.as_ref().display())) +} + +/// Wrap `fs::create_dir_all` with a nicer error message +pub fn create_dir_all>(path: P) -> Result<()> { + fs::create_dir_all(&path) + .chain_err(|| format!("failed to create dir '{}'", path.as_ref().display())) +} + +/// Wrap `remove_dir_all` with a nicer error message +pub fn remove_dir_all>(path: P) -> Result<()> { + ::remove_dir_all::remove_dir_all(path.as_ref()) + .chain_err(|| format!("failed to remove dir '{}'", path.as_ref().display())) +} + +/// Wrap `fs::remove_file` with a nicer error message +pub fn remove_file>(path: P) -> Result<()> { + fs::remove_file(path.as_ref()) + .chain_err(|| format!("failed to remove file '{}'", path.as_ref().display())) +} + +/// Wrap `fs::rename` with a nicer error message +pub fn rename, Q: AsRef>(from: P, to: Q) -> Result<()> { + fs::rename(&from, &to) + .chain_err(|| format!("failed to rename '{}' to '{}'", + from.as_ref().display(), to.as_ref().display())) +} + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. -pub fn copy_recursive(src: &Path, dst: &Path) -> io::Result<()> { +pub fn copy_recursive(src: &Path, dst: &Path) -> Result<()> { copy_with_callback(src, dst, |_, _| Ok(())) } /// Copies the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. Invokes a callback for each path visited. -pub fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> io::Result<()> - where F: FnMut(&Path, fs::FileType) -> io::Result<()> +pub fn copy_with_callback(src: &Path, dst: &Path, mut callback: F) -> Result<()> + where F: FnMut(&Path, fs::FileType) -> Result<()> { for entry in WalkDir::new(src).min_depth(1) { let entry = entry?; let file_type = entry.file_type(); - let path = entry.path().strip_prefix(src).unwrap(); + let path = entry.path().strip_prefix(src)?; let dst = dst.join(path); if file_type.is_dir() { - fs::create_dir(&dst)?; + create_dir(&dst)?; } else { - fs::copy(entry.path(), dst)?; + copy(entry.path(), dst)?; } callback(&path, file_type)?; } From f74789f530c95fa41f4ee5c949bc5d80bc806a75 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 16:08:10 -0700 Subject: [PATCH 19/21] Standardize File errors --- src/combiner.rs | 19 ++++++++----------- src/generator.rs | 13 ++++++------- src/scripter.rs | 14 +++----------- src/tarballer.rs | 12 +++--------- src/util.rs | 26 ++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/combiner.rs b/src/combiner.rs index 512d9d5..d084a85 100644 --- a/src/combiner.rs +++ b/src/combiner.rs @@ -8,7 +8,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::fs; use std::io::{Read, Write}; use std::path::Path; use flate2::read::GzDecoder; @@ -63,12 +62,10 @@ impl Combiner { create_dir_all(&package_dir)?; // Merge each installer into the work directory of the new installer - let components = fs::File::create(package_dir.join("components")) - .chain_err(|| "failed to create a new components file")?; + let components = create_new_file(package_dir.join("components"))?; for input_tarball in self.input_tarballs.split(',').map(str::trim).filter(|s| !s.is_empty()) { // Extract the input tarballs - fs::File::open(&input_tarball) - .and_then(GzDecoder::new) + GzDecoder::new(open_file(&input_tarball)?) .and_then(|tar| Archive::new(tar).unpack(&self.work_dir)) .chain_err(|| format!("unable to extract '{}' into '{}'", &input_tarball, self.work_dir))?; @@ -79,8 +76,8 @@ impl Combiner { // Verify the version number let mut version = String::new(); - fs::File::open(pkg_dir.join("rust-installer-version")) - .and_then(|mut file| file.read_to_string(&mut version)) + open_file(pkg_dir.join("rust-installer-version")) + .and_then(|mut file| file.read_to_string(&mut version).map_err(Error::from)) .chain_err(|| format!("failed to read version in '{}'", input_tarball))?; if version.trim().parse() != Ok(::RUST_INSTALLER_VERSION) { bail!("incorrect installer version in {}", input_tarball); @@ -88,8 +85,8 @@ impl Combiner { // Move components to the new combined installer let mut pkg_components = String::new(); - fs::File::open(pkg_dir.join("components")) - .and_then(|mut file| file.read_to_string(&mut pkg_components)) + open_file(pkg_dir.join("components")) + .and_then(|mut file| file.read_to_string(&mut pkg_components).map_err(Error::from)) .chain_err(|| format!("failed to read components in '{}'", input_tarball))?; for component in pkg_components.split_whitespace() { // All we need to do is move the component directory @@ -104,8 +101,8 @@ impl Combiner { drop(components); // Write the installer version - fs::File::create(package_dir.join("rust-installer-version")) - .and_then(|file| writeln!(&file, "{}", ::RUST_INSTALLER_VERSION)) + let version = package_dir.join("rust-installer-version"); + writeln!(create_new_file(version)?, "{}", ::RUST_INSTALLER_VERSION) .chain_err(|| "failed to write new installer version")?; // Copy the overlay diff --git a/src/generator.rs b/src/generator.rs index e76e809..f5be602 100644 --- a/src/generator.rs +++ b/src/generator.rs @@ -8,7 +8,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::fs; use std::io::Write; use std::path::Path; @@ -71,14 +70,14 @@ impl Generator { copy_and_manifest(self.image_dir.as_ref(), &component_dir, &self.bulk_dirs)?; // Write the component name - fs::File::create(package_dir.join("components")) - .and_then(|file| writeln!(&file, "{}", self.component_name)) + let components = package_dir.join("components"); + writeln!(create_new_file(components)?, "{}", self.component_name) .chain_err(|| "failed to write the component file")?; // Write the installer version (only used by combine-installers.sh) - fs::File::create(package_dir.join("rust-installer-version")) - .and_then(|file| writeln!(&file, "{}", ::RUST_INSTALLER_VERSION)) - .chain_err(|| "failed to write the installer version")?; + let version = package_dir.join("rust-installer-version"); + writeln!(create_new_file(version)?, "{}", ::RUST_INSTALLER_VERSION) + .chain_err(|| "failed to write new installer version")?; // Copy the overlay if !self.non_installed_overlay.is_empty() { @@ -110,7 +109,7 @@ impl Generator { /// Copies the `src` directory recursively to `dst`, writing `manifest.in` too. fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> Result<()> { - let manifest = fs::File::create(dst.join("manifest.in"))?; + let manifest = create_new_file(dst.join("manifest.in"))?; let bulk_dirs: Vec<_> = bulk_dirs.split(',') .filter(|s| !s.is_empty()) .map(Path::new).collect(); diff --git a/src/scripter.rs b/src/scripter.rs index 9380424..66d7ba5 100644 --- a/src/scripter.rs +++ b/src/scripter.rs @@ -8,15 +8,10 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::fs; use std::io::Write; -// Needed to set the script mode to executable. -#[cfg(unix)] -use std::os::unix::fs::OpenOptionsExt; -// FIXME: what about Windows? Are default ACLs executable? - use errors::*; +use util::*; const TEMPLATE: &'static str = include_str!("../install-template.sh"); @@ -59,11 +54,8 @@ impl Scripter { .replace("%%TEMPLATE_LEGACY_MANIFEST_DIRS%%", &sh_quote(&self.legacy_manifest_dirs)) .replace("%%TEMPLATE_RUST_INSTALLER_VERSION%%", &sh_quote(&::RUST_INSTALLER_VERSION)); - let mut options = fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] options.mode(0o755); - options.open(&self.output_script) - .and_then(|mut output| output.write_all(script.as_ref())) + create_new_executable(&self.output_script)? + .write_all(script.as_ref()) .chain_err(|| format!("failed to write output script '{}'", self.output_script)) } } diff --git a/src/tarballer.rs b/src/tarballer.rs index 9f9fbcc..0aec8ae 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -8,7 +8,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use std::fs; use std::io::{self, Write}; use std::path::Path; @@ -56,14 +55,10 @@ impl Tarballer { files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev())); // Prepare the .tar.gz file - let output = fs::File::create(&tar_gz) - .chain_err(|| "failed to create .tar.gz file")?; - let gz = GzEncoder::new(output, flate2::Compression::Best); + let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::Best); // Prepare the .tar.xz file - let output = fs::File::create(&tar_xz) - .chain_err(|| "failed to create .tar.xz file")?; - let xz = XzEncoder::new(output, 9); + let xz = XzEncoder::new(create_new_file(tar_xz)?, 9); // Write the tar into both encoded files. We write all directories // first, so files may be directly created. (see rustup.rs#1092) @@ -75,8 +70,7 @@ impl Tarballer { } for path in files { let src = Path::new(&self.work_dir).join(&path); - fs::File::open(&src) - .and_then(|mut file| builder.append_file(&path, &mut file)) + builder.append_file(&path, &mut open_file(&src)?) .chain_err(|| format!("failed to tar file '{}'", src.display()))?; } let Tee(gz, xz) = builder.into_inner() diff --git a/src/util.rs b/src/util.rs index 22df766..acc269a 100644 --- a/src/util.rs +++ b/src/util.rs @@ -13,6 +13,11 @@ use std::fs; use std::path::Path; use walkdir::WalkDir; +// Needed to set the script mode to executable. +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +// FIXME: what about Windows? Are default ACLs executable? + use errors::*; /// Convert a `&Path` to a UTF-8 `&str` @@ -41,6 +46,27 @@ pub fn create_dir_all>(path: P) -> Result<()> { .chain_err(|| format!("failed to create dir '{}'", path.as_ref().display())) } +/// Wrap `fs::OpenOptions::create_new().open()` as executable, with a nicer error message +pub fn create_new_executable>(path: P) -> Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] options.mode(0o755); + options.open(&path) + .chain_err(|| format!("failed to create file '{}'", path.as_ref().display())) +} + +/// Wrap `fs::OpenOptions::create_new().open()`, with a nicer error message +pub fn create_new_file>(path: P) -> Result { + fs::OpenOptions::new().write(true).create_new(true).open(&path) + .chain_err(|| format!("failed to create file '{}'", path.as_ref().display())) +} + +/// Wrap `fs::File::open()` with a nicer error message +pub fn open_file>(path: P) -> Result { + fs::File::open(&path) + .chain_err(|| format!("failed to open file '{}'", path.as_ref().display())) +} + /// Wrap `remove_dir_all` with a nicer error message pub fn remove_dir_all>(path: P) -> Result<()> { ::remove_dir_all::remove_dir_all(path.as_ref()) From 80a42822c48ad0cb09d6f0b967ac414e7a1abbf3 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 May 2017 16:37:53 -0700 Subject: [PATCH 20/21] Include the root directory in tarball --- src/tarballer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index 0aec8ae..48bac68 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -97,7 +97,7 @@ fn get_recursive_paths(root: P, name: Q) -> Result<(Vec, Vec Date: Fri, 5 May 2017 21:40:11 -0700 Subject: [PATCH 21/21] Tell Travis that we're rusty now --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 64a9a9a..bcc3999 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,4 @@ -addons: - apt: - packages: - - p7zip-full - +language: rust script: + - cargo build - ./test.sh