Straightforward example implementing operables/containers?

[NotBrianZach]

my apologies if I missed it, but it would be nice if there was some code linked in the readme like std-book that also implemented operables/containers for newbs like me that you could just copy&paste and iterate on

this video had some good examples but I'm not sure where the code they used in the video is store
https://www.loom.com/share/27d91aa1eac24bcaaaed18ea6d6d03ca

found this via https://github.com/search?q=%22divnix%2Fstd%22+path%3Aflake.nix+operables&type=code

still not tons of examples for that search

(e.g. I am just trying to run postgresql in a container in a "std" way)

[blaggacao]

I think after landing #297, I'd be at least able to informally share some more, including arion integration. I'll probably repost a snippet here, for a "quick-fix", well aware that this ticket rightfully asks for a larger body of examples.

[blaggacao]

Ok, so let me share this example:

❯ l nix/cardano-stack # the Cardano Stack Software System / Cell
Permissions Size User      Date Modified Name
drwxrwxr-x     - blaggacao 17 May 17:35  deployments
.rw-rw-r--  4.1k blaggacao 29 May 12:37  deployments.nix
.rw-rw-r--  2.3k blaggacao 29 May 12:37  entrypoints.nix
.rw-rw-r--   585 blaggacao 29 May 12:37  oci-images.nix
.rw-rw-r--   108 blaggacao 31 May 12:54  packages.nix
.rw-rw-r--  1.5k blaggacao 29 May 12:37  testbed.nix

---

# ./packages.nix
# in this case, packages are packaged in upstream flake
{
  inherit
    (inputs.offchain-metadata-tools.app.packages)
    metadata-server
    metadata-sync
    ;
}

---

# ./entrypoints.nix  # could also call them `operables.nix`
# tries to capture the runtime contract explicitly in a format
# that is easy to communicate (bash)
let
  inherit (inputs) std nixpkgs;
  dbConnectionEnv = ''
    if [[ -z "''${DB_NAME:-}" ]]; then
      echo DB_NAME must be explicitly set
      exit 1
    fi
    echo "Metadata will be stored in the ''${DB_NAME} database..."
    if [[ -z "''${DB_USER:-}" ]]; then
      echo DB_USER must be explicitly set
      exit 1
    fi
    echo "Metadata will be accessed via the ''${DB_USER} database user..."
    if [[ -z "''${DB_PASS:-}" ]]; then
      echo DB_PASS must be explicitly set
      exit 1
    fi
    if [[ -z "''${DB_HOST:-}" ]]; then
      echo DB_HOST must be explicitly set
      exit 1
    fi
    echo "Metadata will be accessed via the ''${DB_HOST} database host..."
    if [[ -z "''${DB_PORT:-}" ]]; then
      echo DB_PORT must be explicitly set
      exit 1
    fi
    echo "Metadata will be accessed via the ''${DB_PORT} database host..."
  '';
in {
  metadata-server = std.lib.ops.mkOperable {
    package = cell.packages.metadata-server;
    runtimeScript = ''
      ${dbConnectionEnv}
      if [[ -z "''${PORT:-}" ]]; then
        echo PORT must be explicitly set
        exit 1
      fi
      echo "Metadata will be served on port ''${PORT}..."
      exec ${cell.packages.metadata-server}/bin/metadata-server \
        --db "$DB_NAME" \
        --db-user "$DB_USER" \
        --db-pass "$DB_PASS" \
        --db-host "$DB_HOST" \
        --db-port "$DB_PORT" \
        --port "$PORT"
    '';
  };
  metadata-sync = std.lib.ops.mkOperable {
    package = cell.packages.metadata-sync;
    runtimeInputs = [nixpkgs.gitMinimal];
    runtimeScript = ''
      ${dbConnectionEnv}
      if [[ -z "''${GIT_URL:-}" ]]; then
        echo GIT_URL must be explicitly set
        exit 1
      fi
      echo "Metadata will be sync from ''${GIT_URL}..."
      if [[ -z "''${GIT_METADATA_FOLDER:-}" ]]; then
        echo GIT_METADATA_FOLDER must be explicitly set
        exit 1
      fi
      echo "Metadata will be sync from the folder ''${GIT_METADATA_FOLDER}..."
      exec ${cell.packages.metadata-sync}/bin/metadata-sync \
        --db "$DB_NAME" \
        --db-user "$DB_USER" \
        --db-pass "$DB_PASS" \
        --db-host "$DB_HOST" \
        --db-port "$DB_PORT" \
        --git-url "$GIT_URL" \
        --git-metadata-folder "$GIT_METADATA_FOLDER"
    '';
  };
}

---

# ./oci-images.nix
# simple case: just wraps the operable and adds metadata
let
  inherit (inputs) std;
in {
  metadata-server = std.lib.ops.mkStandardOCI {
    name = "****.amazonaws.com/metadata-server";
    operable = cell.entrypoints.metadata-server;
    meta = {
      description = "The metadata server API.";
    };
  };
  metadata-sync = std.lib.ops.mkStandardOCI {
    name = "****.aamazonaws.com/metadata-sync";
    operable = cell.entrypoints.metadata-sync;
    meta = {
      description = "A component that syncs metadata from the Registry (GitHub Repo) to the Sink (Postgres DB).";
    };
  };
}

---

# ./deployments.nix
# a prototype implementation using the new haumea matchers
# this generates k8s manifests
# TODO: stabilize and upstream
let
  domain = "eks.lw.iog.io";

  inherit (inputs) haumea;
  inherit (inputs.std) dmerge;

  inherit (inputs.nixpkgs) runCommand remarshal;

  # Read a YAML file into a Nix datatype using IFD.
  # Similar to:
  # > builtins.fromJSON (builtins.readFile ./somefile)
  # but takes an input file in YAML instead of JSON.
  #
  # Type:
  #   Path -> a :: Nix
  readYAML = path: let
    jsonOutputDrv =
      runCommand "from-yaml"
      {nativeBuildInputs = [remarshal];}
      "remarshal -if yaml -i \"${path}\" -of json -o \"$out\"";
  in
    fromJSON (readFile jsonOutputDrv);

  inherit
    (builtins)
    fromJSON
    ;
  inherit
    (inputs.nixpkgs.lib)
    attrNames
    elemAt
    foldl'
    functionArgs
    isFunction
    isAttrs
    length
    mapAttrs
    mapAttrsRecursiveCond
    optionalAttrs
    pipe
    readFile
    setFunctionArgs
    mutuallyExclusive
    subtractLists
    traceSeq
    generators
    ;

  mkNames = matches: rec {
    env = elemAt matches 0;
    name = release + "-backend";
    namespace = env + "-" + network;
    network = elemAt matches 1;
    region = elemAt matches 2;
    release = namespace + "-cardanojs";
  };

  isWrappedComponent = as: as ? __initNomenclature;
  loadComponent = f: nomenclature: pipe f [functionArgs (mapAttrs (name: _: nomenclature.${name})) f];
  toComponent = f: let
    sig1 = attrNames (mkNames null);
    sig2 = attrNames (functionArgs f);
    excess = subtractLists sig1 sig2;
    ok =
      isFunction f
      && (! mutuallyExclusive sig2 sig1)
      && (
        if excess == []
        then true
        else
          abort ''

            Nomenclature currying function signature
              ${generators.toPretty {multiline = false;} sig2}
            has more elements than the available nomenclature
              ${generators.toPretty {multiline = false;} sig1}.
          ''
      );
    wrapped = setFunctionArgs f (functionArgs f) // {__initNomenclature = true;};
  in (
    if ok
    then wrapped
    else f
  );

  instantiateLeaves = nomenclature:
    mapAttrsRecursiveCond (c: (!isWrappedComponent c))
    (p: f:
      if isWrappedComponent f
      then loadComponent f nomenclature
      else f);

  mkComponents = root: nomenclature: let
    inherit (dmerge) chainMerge chainable;
    components = instantiateLeaves nomenclature root.components;
  in {
    WithBase = chainable root.base;
    WithRegion = chainable components.Region;
    WithNamespace = chainable components.Namespace;
    WithCardanoStack = chainable components.CardanoStack;
  };

  loadEnvimontentNix = matches: args: let
    nomenclature = mkNames matches;
    Components = mkComponents args.root nomenclature;
  in
    haumea.lib.loaders.default {
      inherit domain;
      inherit (inputs.nixpkgs) lib;
      inherit (args) root;
      inherit (dmerge) update append updateOn chainMerge;
      inherit Components;
    };

  loadYaml = _: _: readYAML;

  loadMaybeComponent = _: args: path: let
    f =
      haumea.lib.loaders.default {
        inherit domain;
        inherit (args) root;
        inherit (dmerge) update append updateOn;
      }
      path;
  in
    toComponent f;

  inherit (haumea.lib.transformers) liftDefault;
in
  haumea.lib.load {
    src = ./deployments;
    transformer = liftDefault;
    loader =  [
      (haumea.matchers.regex ''^.+\.(yaml|yml)'' loadYaml)
      (haumea.matchers.regex ''^(.+)-(.+)@(.+)\.nix$'' loadEnvimontentNix)
      (haumea.matchers.always loadMaybeComponent)
    ];
  }

#./deployments/live-preprod@region-a.nix
{
chainMerge,
Components,
}:
with Components;
chainMerge WithBase WithRegion WithNamespace WithCardanoStack {
  meta.description = "Live Environment (user-facing) on the Cardano Proprod Chain (Region A)";
  templates = {
    backend-deployment = {spec.replicas = 1;};
  };
}

[blaggacao]

Oh and the testbed.nix: (which could benefit from nlewo/nix2container#75)

let
  inherit (inputs) std;
in {
  metadata = let
    user = "metadata";
    pass = "bar";
  in
    std.lib.dev.mkArion {
      project.name = "metadata-testbed";
      services = {
        postgres.service = {
          image = "postgres";
          restart = "always";
          ports = ["5432:5432"];
          environment = {
            POSTGRES_PASSWORD = pass;
            POSTGRES_USER = user;
          };
        };
        metadata-sync = {
          service = {
            useHostStore = true;
            depends_on = ["postgres"];
            image = cell.oci-images.metadata-sync.imageRefUnsafe;
            environment = {
              DB_NAME = user;
              DB_USER = user;
              DB_PASS = pass;
              DB_HOST = "postgres";
              DB_PORT = "5432";
              GIT_URL = "https://github.com/cardano-foundation/cardano-token-registry.git";
              GIT_METADATA_FOLDER = "mappings";
            };
          };
        };
        metadata-server = {
          service = {
            useHostStore = true;
            depends_on = ["postgres"];
            image = cell.oci-images.metadata-server.imageRefUnsafe;
            ports = ["8080:8080"];
            environment = {
              PORT = "8080";
              DB_NAME = user;
              DB_USER = user;
              DB_PASS = pass;
              DB_HOST = "postgres";
              DB_PORT = "5432";
            };
          };
        };
      };
    };
}

[blaggacao]

Each of those can be interacted with via the available actions in std's TUI with the following bock type declaration:

[
        (arion "testbed")
        # Packaging Layers
        (installables "packages") # {ci.build = true;})
        (runnables "entrypoints")
        (containers "oci-images" {ci.publish = true;})
        # Deployments
        (helm "deployments" {ci.diff = true;}) # helm is a repo-local block type that isn't upstreamed (yet)
]

[Pegasust]

How does this look under flakes output? Do we plan on partially supporting flakes output or go with the pure std in the near future?

[blaggacao]

@Pegasust for flake outputs compliance, you can use a layer of soil:

growOn {}
# soil for nix cli compat
{
  # capture targets
  n2cImages = std.harvest inputs.self ["myapp" "oci-images"];
  # captures actions ( didn't try that yet - let me know :-) )
  app = std.harvest inputs.self.__std.actions ["myapp" "oci-images" ];
}

For average end users not familiar with nix CLI, std (or your branded version) may be a better option.

[Pegasust]

Actions capturing is very cool, though flakes' output is flat, so we'll need to massage it a bit at the harvest level, so we'll need to massage. We could definitely add a blocktype for this.

We'll also need to turn type derivation -> app

Here's a dump that should be straight-forward regarding the current state of actions capturing

Config:

apps = inputs.nixpkgs.lib.recursiveUpdate (std.harvest self [["repo" "apps"]]) (std.harvest inputs.self.__std.actions [["ops" "operable"]]);

Yields a repl result like this

#                             vvvvvvvvvvvvvvvvvv (runnables "operable")
nix-repl> apps.aarch64-darwin.racker-backend-ops.build.type
"derivation"

[blaggacao]

This would work too (by just using another layer of soil):

# 1. layer of soil
{
  apps = std.harvest self [["repo" "apps"]];
}
# 2. layer of soil
{
  apps = std.harvest inputs.self.__std.actions [["ops" "operable"]];
}

If action harvesting is of more interest, we could definitely add that (and its shape-hammering) to paisano-nix/core.