From 10cf4a1b0a90787ed50f3153f481e7128cffbec7 Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 20:33:39 -0500 Subject: [PATCH 1/7] cargo: wip --- src/cargo/cargo.rc | 8 +- src/cargo/cargo.rs | 192 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 190 insertions(+), 10 deletions(-) diff --git a/src/cargo/cargo.rc b/src/cargo/cargo.rc index 93d58f6dbad47..54267c570fe34 100644 --- a/src/cargo/cargo.rc +++ b/src/cargo/cargo.rc @@ -2,9 +2,6 @@ // cargo.rs - Rust package manager -use std; -use rustc; - // Local Variables: // fill-column: 78; // indent-tabs-mode: nil @@ -12,3 +9,8 @@ use rustc; // buffer-file-coding-system: utf-8-unix // compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'"; // End: + +#[link(name = "cargo", + vers = "0.1", + uuid = "9ff87a04-8fed-4295-9ff8-f99bb802650b", + url = "http://rust-lang.org/doc/cargo")]; diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index d698011f2989d..ab6493ae77cc0 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -1,5 +1,8 @@ // cargo.rs - Rust package manager +use rustc; +use std; + import rustc::syntax::{ast, codemap, visit}; import rustc::syntax::parse::parser; @@ -9,18 +12,39 @@ import std::io; import std::json; import option; import option::{none, some}; +import result; +import std::map; import std::os; import std::run; import str; import std::tempfile; import vec; +tag _src { + /* Break cycles in package <-> source */ + _source(source); +} + +type package = { + source: _src, + name: str, + uuid: str, + url: str +}; + +type source = { + name: str, + url: str, + mutable packages: [package] +}; + type cargo = { root: str, bindir: str, libdir: str, workdir: str, - fetchdir: str + sourcedir: str, + sources: map::hashmap }; type pkg = { @@ -32,6 +56,14 @@ type pkg = { crate_type: option::t }; +fn info(msg: str) { + io::stdout().write_line(msg); +} + +fn warn(msg: str) { + io::stdout().write_line("warning: " + msg); +} + fn load_link(mis: [@ast::meta_item]) -> (option::t, option::t, option::t) { @@ -118,6 +150,96 @@ fn need_dir(s: str) { } } +fn parse_source(name: str, j: json::json) -> source { + alt j { + json::dict(_j) { + alt _j.find("url") { + some(json::string(u)) { + ret { name: name, url: u, mutable packages: [] }; + } + _ { fail "Needed 'url' field in source."; } + }; + } + _ { fail "Needed dict value in source."; } + }; +} + +fn try_parse_sources(filename: str, sources: map::hashmap) { + if !fs::path_exists(filename) { ret; } + let c = io::read_whole_file_str(filename); + let j = json::from_str(result::get(c)); + alt j { + some(json::dict(_j)) { + _j.items { |k, v| + sources.insert(k, parse_source(k, v)); + log #fmt["source: %s", k]; + } + } + _ { fail "malformed sources.json"; } + } +} + +fn load_one_source_package(c: cargo, src: source, p: map::hashmap) { + let name = alt p.find("name") { + some(json::string(_n)) { _n } + _ { + warn("Malformed source json: " + src.name + " (missing name)"); + ret; + } + }; + + let uuid = alt p.find("uuid") { + some(json::string(_n)) { _n } + _ { + warn("Malformed source json: " + src.name + " (missing uuid)"); + ret; + } + }; + + let url = alt p.find("url") { + some(json::string(_n)) { _n } + _ { + warn("Malformed source json: " + src.name + " (missing url)"); + ret; + } + }; + + vec::grow(src.packages, 1u, { + source: _source(src), + name: name, + uuid: uuid, + url: url + }); + info(" Loaded package: " + src.name + "/" + name); +} + +fn load_source_packages(c: cargo, src: source) { + info("Loading source: " + src.name); + let dir = fs::connect(c.sourcedir, src.name); + let pkgfile = fs::connect(dir, "packages.json"); + if !fs::path_exists(pkgfile) { ret; } + let pkgstr = io::read_whole_file_str(pkgfile); + let j = json::from_str(result::get(pkgstr)); + alt j { + some(json::list(js)) { + for _j: json::json in *js { + alt _j { + json::dict(_p) { + load_one_source_package(c, src, _p); + } + _ { + warn("Malformed source json: " + src.name + " (non-dict pkg)"); + ret; + } + } + } + } + _ { + warn("Malformed source json: " + src.name); + } + }; +} + fn configure() -> cargo { let p = alt generic_os::getenv("CARGO_ROOT") { some(_p) { _p } @@ -129,25 +251,39 @@ fn configure() -> cargo { } }; - log #fmt["p: %s", p]; - + let sources = map::new_str_hash::(); + try_parse_sources(fs::connect(p, "sources.json"), sources); + try_parse_sources(fs::connect(p, "local-sources.json"), sources); let c = { root: p, bindir: fs::connect(p, "bin"), libdir: fs::connect(p, "lib"), workdir: fs::connect(p, "work"), - fetchdir: fs::connect(p, "fetch") + sourcedir: fs::connect(p, "sources"), + sources: sources }; need_dir(c.root); - need_dir(c.fetchdir); + need_dir(c.sourcedir); need_dir(c.workdir); need_dir(c.libdir); need_dir(c.bindir); + sources.values { |v| + load_source_packages(c, v); + }; + c } +fn for_each_package(c: cargo, b: block(package)) { + c.sources.values({ |v| + for p in v.packages { + b(p); + } + }) +} + fn install_one_crate(c: cargo, _path: str, cf: str, _p: pkg) { let name = fs::basename(cf); let ri = str::index(name, '.' as u8); @@ -233,11 +369,29 @@ fn install_resolved(c: cargo, wd: str, key: str) { } fn install_uuid(c: cargo, wd: str, uuid: str) { - install_resolved(c, wd, "by-uuid/" + uuid); + let ps = []; + for_each_package(c, { |p| + if p.uuid == uuid { + vec::grow(ps, 1u, p); + } + }); + info("Found:"); + for p in ps { + info(" " + p.source.name + "/" + p.name); + } } fn install_named(c: cargo, wd: str, name: str) { - install_resolved(c, wd, "by-name/" + name); + let ps = []; + for_each_package(c, { |p| + if p.name == name { + vec::grow(ps, 1u, p); + } + }); + info("Found:"); + for p in ps { + info(" " + p.source.name + "/" + p.name); + } } fn cmd_install(c: cargo, argv: [str]) { @@ -268,6 +422,29 @@ fn cmd_install(c: cargo, argv: [str]) { } } +fn sync_one(c: cargo, name: str, src: source) { + let dir = fs::connect(c.sourcedir, name); + let pkgfile = fs::connect(dir, "packages.json"); + let url = src.url; + need_dir(dir); + let p = run::program_output("curl", ["-f", "-s", "-o", pkgfile, url]); + if p.status != 0 { + warn(#fmt["fetch for source %s (url %s) failed", name, url]); + } else { + info(#fmt["fetched source: %s", name]); + } +} + +fn cmd_sync(c: cargo, argv: [str]) { + if vec::len(argv) == 3u { + sync_one(c, argv[2], c.sources.get(argv[2])); + } else { + c.sources.items { |k, v| + sync_one(c, k, v); + } + } +} + fn cmd_usage() { print("Usage: cargo [args...]"); } @@ -280,6 +457,7 @@ fn main(argv: [str]) { let c = configure(); alt argv[1] { "install" { cmd_install(c, argv); } + "sync" { cmd_sync(c, argv); } "usage" { cmd_usage(); } _ { cmd_usage(); } } From 8b7a41f23d421bb4d0dd0ef33be147dc5d065967 Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:08:25 -0500 Subject: [PATCH 2/7] cargo: Support distributed package indexes. Indexes are listed in ~/.cargo/sources.json and ~/.cargo/local-sources.json, the former of which is stored in the rust source tree in src/cargo. Each entry in either of these files is a source, which is a dictionary with (currently) a single key, "url". The supplied url should point to a json list, each element of which should be a dictionary with four keys: "name", "uuid", "url", and "method". The name and uuid serve to identify the package; the method describes how to fetch the package; the url describes where to fetch it from. Currently supported methods are "git", "http", and "file". Signed-off-by: Elly Jones --- src/cargo/cargo.rs | 166 ++++++++++++++++++++++++++++++++--------- src/cargo/sources.json | 5 ++ 2 files changed, 135 insertions(+), 36 deletions(-) create mode 100644 src/cargo/sources.json diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index ab6493ae77cc0..0445765a77c84 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -26,10 +26,11 @@ tag _src { } type package = { - source: _src, +// source: _src, name: str, uuid: str, - url: str + url: str, + method: str }; type source = { @@ -64,6 +65,10 @@ fn warn(msg: str) { io::stdout().write_line("warning: " + msg); } +fn error(msg: str) { + io::stdout().write_line("error: " + msg); +} + fn load_link(mis: [@ast::meta_item]) -> (option::t, option::t, option::t) { @@ -179,7 +184,7 @@ fn try_parse_sources(filename: str, sources: map::hashmap) { } } -fn load_one_source_package(c: cargo, src: source, p: map::hashmap) { +fn load_one_source_package(&c: cargo, &src: source, p: map::hashmap) { let name = alt p.find("name") { some(json::string(_n)) { _n } _ { @@ -204,16 +209,25 @@ fn load_one_source_package(c: cargo, src: source, p: map::hashmap cargo { need_dir(c.libdir); need_dir(c.bindir); - sources.values { |v| - load_source_packages(c, v); + sources.keys { |k| + let s = sources.get(k); + load_source_packages(c, s); + sources.insert(k, s); }; c } -fn for_each_package(c: cargo, b: block(package)) { +fn for_each_package(c: cargo, b: block(source, package)) { c.sources.values({ |v| for p in v.packages { - b(p); + b(v, p); } }) } @@ -335,14 +350,25 @@ fn install_source(c: cargo, path: str) { } } -fn install_git(c: cargo, wd: str, _path: str) { - run::run_program("git", ["clone", _path, wd]); +fn install_git(c: cargo, wd: str, url: str) { + run::run_program("git", ["clone", url, wd]); install_source(c, wd); } -fn install_file(c: cargo, wd: str, _path: str) { +fn install_curl(c: cargo, wd: str, url: str) { + let tarpath = fs::connect(wd, "pkg.tar"); + let p = run::program_output("curl", ["-f", "-s", "-o", + tarpath, url]); + if p.status != 0 { + fail #fmt["Fetch of %s failed: %s", url, p.err]; + } run::run_program("tar", ["-x", "--strip-components=1", - "-C", wd, "-f", _path]); + "-C", wd, "-f", tarpath]); +} + +fn install_file(c: cargo, wd: str, path: str) { + run::run_program("tar", ["-x", "--strip-components=1", + "-C", wd, "-f", path]); install_source(c, wd); } @@ -368,30 +394,90 @@ fn install_resolved(c: cargo, wd: str, key: str) { } } +fn install_package(c: cargo, wd: str, pkg: package) { + info("Installing with " + pkg.method + " from " + pkg.url + "..."); + if pkg.method == "git" { + install_git(c, wd, pkg.url); + } else if pkg.method == "http" { + install_curl(c, wd, pkg.url); + } else if pkg.method == "file" { + install_file(c, wd, pkg.url); + } +} + fn install_uuid(c: cargo, wd: str, uuid: str) { let ps = []; - for_each_package(c, { |p| + for_each_package(c, { |s, p| + info(#fmt["%s ? %s", p.uuid, uuid]); if p.uuid == uuid { - vec::grow(ps, 1u, p); + vec::grow(ps, 1u, (s, p)); } }); - info("Found:"); - for p in ps { - info(" " + p.source.name + "/" + p.name); + if vec::len(ps) == 1u { + let (s, p) = ps[0]; + install_package(c, wd, p); + ret; + } else if vec::len(ps) == 0u { + error("No packages."); + ret; + } + error("Found multiple packages:"); + for (s,p) in ps { + info(" " + s.name + "/" + p.uuid + " (" + p.name + ")"); } } fn install_named(c: cargo, wd: str, name: str) { let ps = []; - for_each_package(c, { |p| + for_each_package(c, { |s, p| if p.name == name { - vec::grow(ps, 1u, p); + vec::grow(ps, 1u, (s, p)); } }); - info("Found:"); - for p in ps { - info(" " + p.source.name + "/" + p.name); + if vec::len(ps) == 1u { + let (s, p) = ps[0]; + install_package(c, wd, p); + ret; + } else if vec::len(ps) == 0u { + error("No packages."); + ret; + } + error("Found multiple packages:"); + for (s,p) in ps { + info(" " + s.name + "/" + p.uuid + " (" + p.name + ")"); + } +} + +fn install_uuid_specific(c: cargo, wd: str, src: str, uuid: str) { + alt c.sources.find(src) { + some(s) { + if vec::any(s.packages, { |p| + if p.uuid == uuid { + install_package(c, wd, p); + ret true; + } + ret false; + }) { ret; } + } + _ { } } + error("Can't find package " + src + "/" + uuid); +} + +fn install_named_specific(c: cargo, wd: str, src: str, name: str) { + alt c.sources.find(src) { + some(s) { + if vec::any(s.packages, { |p| + if p.name == name { + install_package(c, wd, p); + ret true; + } + ret false; + }) { ret; } + } + _ { } + } + error("Can't find package " + src + "/" + name); } fn cmd_install(c: cargo, argv: [str]) { @@ -406,19 +492,26 @@ fn cmd_install(c: cargo, argv: [str]) { none. { fail "needed temp dir"; } }; - if str::starts_with(argv[2], "git:") { - install_git(c, wd, argv[2]); - } else if str::starts_with(argv[2], "github:") { - let path = rest(argv[2], 7u); - install_git(c, wd, "git://github.com/" + path); - } else if str::starts_with(argv[2], "file:") { - let path = rest(argv[2], 5u); - install_file(c, wd, path); - } else if str::starts_with(argv[2], "uuid:") { + if str::starts_with(argv[2], "uuid:") { let uuid = rest(argv[2], 5u); - install_uuid(c, wd, uuid); + let idx = str::index(uuid, '/' as u8); + if idx != -1 { + let source = str::slice(uuid, 0u, idx as uint); + uuid = str::slice(uuid, idx as uint + 1u, str::byte_len(uuid)); + install_uuid_specific(c, wd, source, uuid); + } else { + install_uuid(c, wd, uuid); + } } else { - install_named(c, wd, argv[2]); + let name = argv[2]; + let idx = str::index(name, '/' as u8); + if idx != -1 { + let source = str::slice(name, 0u, idx as uint); + name = str::slice(name, idx as uint + 1u, str::byte_len(name)); + install_named_specific(c, wd, source, name); + } else { + install_named(c, wd, name); + } } } @@ -427,6 +520,7 @@ fn sync_one(c: cargo, name: str, src: source) { let pkgfile = fs::connect(dir, "packages.json"); let url = src.url; need_dir(dir); + info(#fmt["fetching source %s...", name]); let p = run::program_output("curl", ["-f", "-s", "-o", pkgfile, url]); if p.status != 0 { warn(#fmt["fetch for source %s (url %s) failed", name, url]); diff --git a/src/cargo/sources.json b/src/cargo/sources.json new file mode 100644 index 0000000000000..c76f133e8f21b --- /dev/null +++ b/src/cargo/sources.json @@ -0,0 +1,5 @@ +{ + "elly": { + "url": "https://raw.github.com/elly/rust-packages/master/packages.json" + } +} From 327af5b9f3cd9ec6963b3c75fec8510a23916750 Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:24:01 -0500 Subject: [PATCH 3/7] cargo: flesh out usage --- src/cargo/cargo.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index 0445765a77c84..2ed16273c3201 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -541,6 +541,10 @@ fn cmd_sync(c: cargo, argv: [str]) { fn cmd_usage() { print("Usage: cargo [args...]"); + print(" install [source/]package-name Install by name"); + print(" install uuid:[source/]package-uuid Install by uuid"); + print(" sync Sync all sources"); + print(" usage This"); } fn main(argv: [str]) { From 9a0b89b5341c93a071e8a5be78b3fb066b12c5ba Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:27:04 -0500 Subject: [PATCH 4/7] cargo: fix some warnings --- src/cargo/cargo.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index 2ed16273c3201..f7b3cd2116829 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -184,7 +184,7 @@ fn try_parse_sources(filename: str, sources: map::hashmap) { } } -fn load_one_source_package(&c: cargo, &src: source, p: map::hashmap) { +fn load_one_source_package(&src: source, p: map::hashmap) { let name = alt p.find("name") { some(json::string(_n)) { _n } _ { @@ -239,7 +239,7 @@ fn load_source_packages(&c: cargo, &src: source) { for _j: json::json in *js { alt _j { json::dict(_p) { - load_one_source_package(c, src, _p); + load_one_source_package(src, _p); } _ { warn("Malformed source json: " + src.name + " (non-dict pkg)"); @@ -364,6 +364,7 @@ fn install_curl(c: cargo, wd: str, url: str) { } run::run_program("tar", ["-x", "--strip-components=1", "-C", wd, "-f", tarpath]); + install_source(c, wd); } fn install_file(c: cargo, wd: str, path: str) { @@ -414,7 +415,7 @@ fn install_uuid(c: cargo, wd: str, uuid: str) { } }); if vec::len(ps) == 1u { - let (s, p) = ps[0]; + let (_, p) = ps[0]; install_package(c, wd, p); ret; } else if vec::len(ps) == 0u { @@ -435,7 +436,7 @@ fn install_named(c: cargo, wd: str, name: str) { } }); if vec::len(ps) == 1u { - let (s, p) = ps[0]; + let (_, p) = ps[0]; install_package(c, wd, p); ret; } else if vec::len(ps) == 0u { From 7bd003a1d0a992a2d70204095e454773f7afe37d Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:27:47 -0500 Subject: [PATCH 5/7] cargo: remove obsolete rust-pkg-index lookup code --- src/cargo/cargo.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index f7b3cd2116829..786e595708a0f 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -373,28 +373,6 @@ fn install_file(c: cargo, wd: str, path: str) { install_source(c, wd); } -fn install_resolved(c: cargo, wd: str, key: str) { - fs::remove_dir(wd); - let u = "https://rust-package-index.appspot.com/pkg/" + key; - let p = run::program_output("curl", [u]); - if p.status != 0 { - fail #fmt["Fetch of %s failed: %s", u, p.err]; - } - let j = json::from_str(p.out); - alt j { - some (json::dict(_j)) { - alt _j.find("install") { - some (json::string(g)) { - log #fmt["Resolved: %s -> %s", key, g]; - cmd_install(c, ["cargo", "install", g]); - } - _ { fail #fmt["Bogus install: '%s'", p.out]; } - } - } - _ { fail #fmt["Bad json: '%s'", p.out]; } - } -} - fn install_package(c: cargo, wd: str, pkg: package) { info("Installing with " + pkg.method + " from " + pkg.url + "..."); if pkg.method == "git" { From b7e30bc4c5852fb7f840f2abd472fa1e68dcfd52 Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:31:49 -0500 Subject: [PATCH 6/7] cargo: fix remaining warnings --- src/cargo/cargo.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index 786e595708a0f..4fc444f853e3b 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -293,7 +293,7 @@ fn configure() -> cargo { fn for_each_package(c: cargo, b: block(source, package)) { c.sources.values({ |v| - for p in v.packages { + for p in copy v.packages { b(v, p); } }) @@ -430,7 +430,7 @@ fn install_named(c: cargo, wd: str, name: str) { fn install_uuid_specific(c: cargo, wd: str, src: str, uuid: str) { alt c.sources.find(src) { some(s) { - if vec::any(s.packages, { |p| + if vec::any(copy s.packages, { |p| if p.uuid == uuid { install_package(c, wd, p); ret true; @@ -446,7 +446,7 @@ fn install_uuid_specific(c: cargo, wd: str, src: str, uuid: str) { fn install_named_specific(c: cargo, wd: str, src: str, name: str) { alt c.sources.find(src) { some(s) { - if vec::any(s.packages, { |p| + if vec::any(copy s.packages, { |p| if p.name == name { install_package(c, wd, p); ret true; From c8427e4ffd59ba6e6347ece491aca43b17207c2e Mon Sep 17 00:00:00 2001 From: Elly Jones Date: Fri, 16 Dec 2011 22:39:33 -0500 Subject: [PATCH 7/7] cargo: fix lib detection logic --- src/cargo/cargo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cargo/cargo.rs b/src/cargo/cargo.rs index 4fc444f853e3b..a982d7473aab2 100644 --- a/src/cargo/cargo.rs +++ b/src/cargo/cargo.rs @@ -314,7 +314,7 @@ fn install_one_crate(c: cargo, _path: str, cf: str, _p: pkg) { let exec_suffix = os::exec_suffix(); for ct: str in created { if (exec_suffix != "" && str::ends_with(ct, exec_suffix)) || - (exec_suffix == "" && !str::starts_with(ct, "lib")) { + (exec_suffix == "" && !str::starts_with(ct, "./lib")) { log #fmt[" bin: %s", ct]; // FIXME: need libstd fs::copy or something run::run_program("cp", [ct, c.bindir]);