Merge pull request #456 from itowlson/wasm-bindle-parcel-source

Allow parcel ref in component source

Author: itowlson

Cargo.lock

@@ -44,6 +44,7 @@ wasmtime = "0.34"
 
 [dev-dependencies]
 hyper = { version = "0.14", features = [ "full" ] }
+sha2 = "0.10.1"
 
 [build-dependencies]
 cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "b7b1989fe0984c0f7c4966398304c6538e52fe49" }

Cargo.toml

@@ -0,0 +1,66 @@
+use std::sync::Arc;
+
+use bindle::client::{
+    tokens::{HttpBasic, NoToken, TokenManager},
+    Client, ClientBuilder,
+};
+
+/// BindleConnectionInfo holds the details of a connection to a
+/// Bindle server, including url, insecure configuration and an
+/// auth token manager
+#[derive(Clone)]
+pub struct BindleConnectionInfo {
+    base_url: String,
+    allow_insecure: bool,
+    token_manager: AnyAuth,
+}
+
+impl BindleConnectionInfo {
+    /// Generates a new BindleConnectionInfo instance using the provided
+    /// base_url, allow_insecure setting and optional username and password
+    /// for basic http auth
+    pub fn new<I: Into<String>>(
+        base_url: I,
+        allow_insecure: bool,
+        username: Option<String>,
+        password: Option<String>,
+    ) -> Self {
+        let token_manager: Box<dyn TokenManager + Send + Sync> = match (username, password) {
+            (Some(u), Some(p)) => Box::new(HttpBasic::new(&u, &p)),
+            _ => Box::new(NoToken::default()),
+        };
+
+        Self {
+            base_url: base_url.into(),
+            allow_insecure,
+            token_manager: AnyAuth {
+                token_manager: Arc::new(token_manager),
+            },
+        }
+    }
+
+    /// Returns a client based on this instance's configuration
+    pub fn client(&self) -> bindle::client::Result<Client<AnyAuth>> {
+        let builder = ClientBuilder::default()
+            .http2_prior_knowledge(false)
+            .danger_accept_invalid_certs(self.allow_insecure);
+        builder.build(&self.base_url, self.token_manager.clone())
+    }
+}
+
+/// AnyAuth wraps an authentication token manager which applies
+/// the appropriate auth header per its configuration
+#[derive(Clone)]
+pub struct AnyAuth {
+    token_manager: Arc<Box<dyn TokenManager + Send + Sync>>,
+}
+
+#[async_trait::async_trait]
+impl TokenManager for AnyAuth {
+    async fn apply_auth_header(
+        &self,
+        builder: reqwest::RequestBuilder,
+    ) -> bindle::client::Result<reqwest::RequestBuilder> {
+        self.token_manager.apply_auth_header(builder).await
+    }
+}

crates/loader/src/bindle/connection.rs

@@ -6,26 +6,26 @@
 mod assets;
 /// Configuration representation for a Spin application in Bindle.
 pub mod config;
+mod connection;
 /// Bindle helper functions.
 mod utils;
 
 use crate::bindle::{
     config::{RawAppManifest, RawComponentManifest},
-    utils::{find_manifest, parcels_in_group, BindleReader},
+    utils::{find_manifest, parcels_in_group},
 };
 use anyhow::{anyhow, Context, Result};
-use bindle::{
-    client::{tokens::NoToken, Client},
-    Invoice,
-};
+use bindle::Invoice;
+pub use connection::BindleConnectionInfo;
 use futures::future;
 use spin_manifest::{
     Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource,
     SpinVersion, WasmConfig,
 };
 use std::{path::Path, sync::Arc};
 use tracing::log;
-pub use utils::{BindleTokenManager, SPIN_MANIFEST_MEDIA_TYPE};
+pub(crate) use utils::BindleReader;
+pub use utils::SPIN_MANIFEST_MEDIA_TYPE;
 
 /// Given a Bindle server URL and reference, pull it, expand its assets locally, and get a
 /// prepared application configuration consumable by a Spin execution context.
@@ -34,8 +34,8 @@ pub use utils::{BindleTokenManager, SPIN_MANIFEST_MEDIA_TYPE};
 pub async fn from_bindle(id: &str, url: &str, base_dst: impl AsRef<Path>) -> Result<Application> {
     // TODO
     // Handle Bindle authentication.
-    let manager = BindleTokenManager::NoToken(NoToken);
-    let client = Client::new(url, manager)?;
+    let connection_info = BindleConnectionInfo::new(url, false, None, None);
+    let client = connection_info.client()?;
     let reader = BindleReader::remote(&client, &id.parse()?);
 
     prepare(id, url, &reader, base_dst).await

crates/loader/src/bindle/mod.rs

@@ -1,23 +1,15 @@
 #![deny(missing_docs)]
 
 use anyhow::{anyhow, bail, Context, Error, Result};
-use async_trait::async_trait;
-use bindle::{
-    client::{
-        self,
-        tokens::{NoToken, TokenManager},
-        Client,
-    },
-    standalone::StandaloneRead,
-    Id, Invoice, Label, Parcel,
-};
+use bindle::{client::Client, standalone::StandaloneRead, Id, Invoice, Label, Parcel};
 use futures::{Stream, StreamExt, TryStreamExt};
 use itertools::Itertools;
-use reqwest::RequestBuilder;
 use std::{fmt::Debug, path::Path, sync::Arc};
 use tokio::fs;
 use tokio_util::codec::{BytesCodec, FramedRead};
 
+use super::connection::AnyAuth;
+
 static EMPTY: &Vec<bindle::Parcel> = &vec![];
 
 // Alternative to storing `spin.toml` as a parcel, this could be
@@ -101,30 +93,6 @@ trigger     = "http"
 
 */
 
-/// Any kind of Bindle authentication.
-#[derive(Clone)]
-pub enum BindleTokenManager {
-    /// Anonymous authentication
-    NoToken(NoToken),
-}
-
-#[async_trait]
-impl TokenManager for BindleTokenManager {
-    async fn apply_auth_header(&self, builder: RequestBuilder) -> client::Result<RequestBuilder> {
-        match self {
-            Self::NoToken(t) => t.apply_auth_header(builder).await,
-        }
-    }
-}
-
-impl Debug for BindleTokenManager {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::NoToken(_) => f.debug_tuple("NoToken").finish(),
-        }
-    }
-}
-
 /// Encapsulate a Bindle source.
 #[derive(Clone, Debug)]
 pub(crate) struct BindleReader {
@@ -207,7 +175,7 @@ impl BindleReader {
         }
     }
 
-    pub(crate) fn remote(c: &Client<BindleTokenManager>, id: &Id) -> Self {
+    pub(crate) fn remote(c: &Client<AnyAuth>, id: &Id) -> Self {
         Self {
             inner: BindleReaderInner::Remote(c.clone(), id.clone()),
         }
@@ -225,7 +193,7 @@ impl BindleReader {
 #[derive(Clone)]
 enum BindleReaderInner {
     Standalone(Arc<StandaloneRead>),
-    Remote(Client<BindleTokenManager>, Id),
+    Remote(Client<AnyAuth>, Id),
 }
 
 impl Debug for BindleReaderInner {

crates/loader/src/bindle/utils.rs

@@ -18,21 +18,27 @@ use spin_manifest::{
     Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource,
     SpinVersion, WasmConfig,
 };
-use std::{path::Path, sync::Arc};
+use std::{path::Path, str::FromStr, sync::Arc};
 use tokio::{fs::File, io::AsyncReadExt};
 
+use crate::bindle::BindleConnectionInfo;
+
 /// Given the path to a spin.toml manifest file, prepare its assets locally and
 /// get a prepared application configuration consumable by a Spin execution context.
 /// If a directory is provided, use it as the base directory to expand the assets,
 /// otherwise create a new temporary directory.
-pub async fn from_file(app: impl AsRef<Path>, base_dst: impl AsRef<Path>) -> Result<Application> {
+pub async fn from_file(
+    app: impl AsRef<Path>,
+    base_dst: impl AsRef<Path>,
+    bindle_connection: &Option<BindleConnectionInfo>,
+) -> Result<Application> {
     let app = app
         .as_ref()
         .absolutize()
         .context("Failed to resolve absolute path to manifest file")?;
     let manifest = raw_manifest_from_file(&app).await?;
 
-    prepare_any_version(manifest, app, base_dst).await
+    prepare_any_version(manifest, app, base_dst, bindle_connection).await
 }
 
 /// Reads the spin.toml file as a raw manifest.
@@ -54,9 +60,10 @@ async fn prepare_any_version(
     raw: RawAppManifestAnyVersion,
     src: impl AsRef<Path>,
     base_dst: impl AsRef<Path>,
+    bindle_connection: &Option<BindleConnectionInfo>,
 ) -> Result<Application> {
     match raw {
-        RawAppManifestAnyVersion::V1(raw) => prepare(raw, src, base_dst).await,
+        RawAppManifestAnyVersion::V1(raw) => prepare(raw, src, base_dst, bindle_connection).await,
     }
 }
 
@@ -79,6 +86,7 @@ async fn prepare(
     mut raw: RawAppManifest,
     src: impl AsRef<Path>,
     base_dst: impl AsRef<Path>,
+    bindle_connection: &Option<BindleConnectionInfo>,
 ) -> Result<Application> {
     let info = info(raw.info, &src);
 
@@ -104,7 +112,7 @@ async fn prepare(
     let components = future::join_all(
         raw.components
             .into_iter()
-            .map(|c| async { core(c, &src, &base_dst).await })
+            .map(|c| async { core(c, &src, &base_dst, bindle_connection).await })
             .collect::<Vec<_>>(),
     )
     .await
@@ -125,7 +133,10 @@ async fn core(
     raw: RawComponentManifest,
     src: impl AsRef<Path>,
     base_dst: impl AsRef<Path>,
+    bindle_connection: &Option<BindleConnectionInfo>,
 ) -> Result<CoreComponent> {
+    let id = raw.id;
+
     let src = src
         .as_ref()
         .parent()
@@ -139,12 +150,33 @@ async fn core(
 
             ModuleSource::FileReference(p)
         }
-        config::RawModuleSource::Bindle(_) => {
-            todo!("Bindle module sources are not yet supported in file-based app config")
+        config::RawModuleSource::Bindle(b) => {
+            let bindle_id = bindle::Id::from_str(&b.reference).with_context(|| {
+                format!("Invalid bindle ID {} in component {}", b.reference, id)
+            })?;
+            let parcel_sha = &b.parcel;
+            let client = match bindle_connection {
+                None => anyhow::bail!(
+                    "Component {} requires a Bindle connection but none was specified",
+                    id
+                ),
+                Some(c) => c.client()?,
+            };
+            let bindle_reader = crate::bindle::BindleReader::remote(&client, &bindle_id);
+            let bytes = bindle_reader
+                .get_parcel(parcel_sha)
+                .await
+                .with_context(|| {
+                    format!(
+                        "Failed to download parcel {}@{} for component {}",
+                        bindle_id, parcel_sha, id
+                    )
+                })?;
+            let name = format!("{}@{}", bindle_id, parcel_sha);
+            ModuleSource::Buffer(bytes, name)
         }
     };
 
-    let id = raw.id;
     let description = raw.description;
     let mounts = match raw.wasm.files {
         Some(f) => assets::prepare_component(&f, src, &base_dst, &id).await?,

crates/loader/src/local/mod.rs

@@ -11,7 +11,7 @@ async fn test_from_local_source() -> Result<()> {
 
     let temp_dir = tempfile::tempdir()?;
     let dir = temp_dir.path();
-    let app = from_file(MANIFEST, dir).await?;
+    let app = from_file(MANIFEST, dir, &None).await?;
 
     assert_eq!(app.info.name, "spin-local-source-test");
     assert_eq!(app.info.version, "1.0.0");
@@ -145,7 +145,7 @@ async fn test_duplicate_component_id_is_rejected() -> Result<()> {
 
     let temp_dir = tempfile::tempdir()?;
     let dir = temp_dir.path();
-    let app = from_file(MANIFEST, dir).await;
+    let app = from_file(MANIFEST, dir, &None).await;
 
     assert!(
         app.is_err(),

crates/loader/src/local/tests.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 
 use anyhow::{bail, Context, Result};
 use clap::{Args, Parser};
+use spin_loader::bindle::BindleConnectionInfo;
 use tempfile::TempDir;
 
 use spin_engine::io::FollowComponents;
@@ -133,7 +134,8 @@ impl UpCommand {
                 let manifest_file = app
                     .as_deref()
                     .unwrap_or_else(|| DEFAULT_MANIFEST_FILE.as_ref());
-                spin_loader::from_file(manifest_file, working_dir).await?
+                let bindle_connection = self.bindle_connection();
+                spin_loader::from_file(manifest_file, working_dir, &bindle_connection).await?
             }
             (None, Some(bindle)) => match &self.server {
                 Some(server) => spin_loader::from_bindle(bindle, server, working_dir).await?,
@@ -222,6 +224,12 @@ impl UpCommand {
             FollowComponents::Named(followed)
         }
     }
+
+    fn bindle_connection(&self) -> Option<BindleConnectionInfo> {
+        self.server
+            .as_ref()
+            .map(|url| BindleConnectionInfo::new(url, false, None, None))
+    }
 }
 
 enum WorkingDirectory {

src/commands/up.rs

@@ -0,0 +1,18 @@
+spin_version = "1"
+authors = ["Fermyon Engineering <[email protected]>"]
+description = "A simple application that returns hello and goodbye."
+name = "spin-hello-from-parcel"
+trigger = {type = "http", base = "/test"}
+version = "1.0.0"
+
+[config]
+object = { default = "teapot" }
+
+[[component]]
+id = "hello"
+source = { reference = "spin-hello-world/1.0.0", parcel = "AWAITING_PARCEL_SHA" }
+files = [ { source = "assets", destination = "/" } ]
+[component.trigger]
+route = "/hello/..."
+[component.config]
+message = "I'm a {{object}}"