From 4f1dc6295095cc12077c0462302c19f416291e6c Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 20 Nov 2018 11:18:11 +0000 Subject: [PATCH 01/15] SSH passthrough uses ssh package Signed-off-by: JoshVanL --- pkg/tarmak/interfaces/interfaces.go | 3 +- pkg/tarmak/ssh.go | 4 +- pkg/tarmak/ssh/ssh.go | 110 +++++++++++++++++++++++++--- 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 13a9a1d370..6efc2a0f85 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -208,7 +208,7 @@ type Terraform interface { type SSH interface { WriteConfig(Cluster) error - PassThrough([]string) + PassThrough([]string) error Tunnel(hostname string, destination string, destinationPort int) Tunnel Execute(host string, cmd string, args []string) (returnCode int, err error) Validate() error @@ -235,6 +235,7 @@ type Host interface { SSHConfig() string Parameters() map[string]string SSHControlPath() string + Aliases() []string } type Puppet interface { diff --git a/pkg/tarmak/ssh.go b/pkg/tarmak/ssh.go index 8711ed76b0..aaaf2eab75 100644 --- a/pkg/tarmak/ssh.go +++ b/pkg/tarmak/ssh.go @@ -18,5 +18,7 @@ func (t *Tarmak) SSHPassThrough(argsAdditional []string) { t.log.Fatal(err) } - t.ssh.PassThrough(argsAdditional) + if err := t.ssh.PassThrough(argsAdditional); err != nil { + t.log.Fatal(err) + } } diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index 4c82dd8618..113062fe7c 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -2,18 +2,24 @@ package ssh import ( + //"bufio" "bytes" "encoding/pem" "errors" "fmt" "io/ioutil" + "net" "os" "os/exec" + // "os/signal" "path/filepath" "syscall" + "time" "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/terminal" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/utils" @@ -78,23 +84,107 @@ func (s *SSH) args() []string { } // Pass through a local CLI session -func (s *SSH) PassThrough(argsAdditional []string) { - args := append(s.args(), argsAdditional...) +func (s *SSH) PassThrough(argsAdditional []string) error { + hosts, err := s.tarmak.Cluster().ListHosts() + if err != nil { + return err + } - cmd := exec.Command(args[0], args[1:len(args)]...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin + var host interfaces.Host + var bastion interfaces.Host + for _, h := range hosts { + if h.Aliases()[0] == argsAdditional[0] { + host = h + continue + } + if h.Aliases()[0] == "bastion" { + bastion = h + } + } - err := cmd.Start() + b, err := ioutil.ReadFile(s.tarmak.Environment().SSHPrivateKeyPath()) if err != nil { - s.log.Fatal(err) + return err + } + signer, err := ssh.ParsePrivateKey(b) + if err != nil { + return err } - err = cmd.Wait() + conf := &ssh.ClientConfig{ + Timeout: time.Minute * 10, + User: host.User(), + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + confProxy := &ssh.ClientConfig{ + Timeout: time.Minute * 10, + User: host.User(), + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), confProxy) + if err != nil { + return fmt.Errorf("failed to set up connection to bastion: %s", err) + } + + conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) + if err != nil { + return fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(host.Hostname(), "22"), conf) + if err != nil { + return fmt.Errorf("failed to set up ssh client: %s", err) + } + + client := ssh.NewClient(ncc, chans, reqs) + sess, err := client.NewSession() if err != nil { - s.log.Fatal(err) + return err + } + defer sess.Close() + + sess.Stderr = os.Stderr + sess.Stdout = os.Stdout + sess.Stdin = os.Stdin + + modes := ssh.TerminalModes{ + ssh.ECHO: 1, + ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud + ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud } + + fileDescriptor := int(os.Stdin.Fd()) + if terminal.IsTerminal(fileDescriptor) { + originalState, err := terminal.MakeRaw(fileDescriptor) + if err != nil { + return err + } + defer terminal.Restore(fileDescriptor, originalState) + + termWidth, termHeight, err := terminal.GetSize(fileDescriptor) + if err != nil { + return err + } + + err = sess.RequestPty("xterm-256color", termHeight, termWidth, modes) + if err != nil { + return err + } + } + + if err := sess.Shell(); err != nil { + return err + } + + if err := sess.Wait(); err != nil { + return err + } + + return nil } func (s *SSH) Execute(host string, command string, argsAdditional []string) (returnCode int, err error) { From 9b41a200f26dce1464a0c8a70aa869f3249ce91f Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 20 Nov 2018 15:51:29 +0000 Subject: [PATCH 02/15] Move Execute to use pkg ssh Signed-off-by: JoshVanL --- cmd/tarmak/cmd/cluster_instances_ssh.go | 18 +-- cmd/tarmak/cmd/cluster_ssh.go | 13 +- pkg/tarmak/environment/bastion.go | 14 +- pkg/tarmak/interfaces/interfaces.go | 4 +- pkg/tarmak/ssh.go | 12 +- pkg/tarmak/ssh/ssh.go | 190 ++++++++++++++---------- pkg/tarmak/ssh/tunnel.go | 8 + 7 files changed, 154 insertions(+), 105 deletions(-) diff --git a/cmd/tarmak/cmd/cluster_instances_ssh.go b/cmd/tarmak/cmd/cluster_instances_ssh.go index 77597bebd8..030a7c8536 100644 --- a/cmd/tarmak/cmd/cluster_instances_ssh.go +++ b/cmd/tarmak/cmd/cluster_instances_ssh.go @@ -1,22 +1,6 @@ // Copyright Jetstack Ltd. See LICENSE for details. package cmd -import ( - "github.com/spf13/cobra" - - "github.com/jetstack/tarmak/pkg/tarmak" -) - -var clusterInstancesSshCmd = &cobra.Command{ - Use: "ssh [instance alias]", - Short: "Log into an instance with SSH", - Run: func(cmd *cobra.Command, args []string) { - t := tarmak.New(globalFlags) - defer t.Cleanup() - t.SSHPassThrough(args) - }, -} - func init() { - clusterInstancesCmd.AddCommand(clusterInstancesSshCmd) + clusterInstancesCmd.AddCommand(clusterSshCmd) } diff --git a/cmd/tarmak/cmd/cluster_ssh.go b/cmd/tarmak/cmd/cluster_ssh.go index debfc5efea..d16c66bd8c 100644 --- a/cmd/tarmak/cmd/cluster_ssh.go +++ b/cmd/tarmak/cmd/cluster_ssh.go @@ -2,18 +2,25 @@ package cmd import ( + "errors" + "github.com/spf13/cobra" "github.com/jetstack/tarmak/pkg/tarmak" ) var clusterSshCmd = &cobra.Command{ - Use: "ssh [instance alias]", + Use: "ssh [instance alias] [optional ssh arguments]", Short: "Log into an instance with SSH", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return errors.New("expecting an instance aliases argument") + } + return nil + }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - defer t.Cleanup() - t.SSHPassThrough(args) + t.Perform(t.SSHPassThrough(args[0], args[1:])) }, } diff --git a/pkg/tarmak/environment/bastion.go b/pkg/tarmak/environment/bastion.go index a718d52a03..602dc5f906 100644 --- a/pkg/tarmak/environment/bastion.go +++ b/pkg/tarmak/environment/bastion.go @@ -2,8 +2,10 @@ package environment import ( + "bufio" "context" "fmt" + "io" "os" "os/signal" "sync" @@ -48,11 +50,19 @@ func (e *Environment) VerifyBastionAvailable() error { expBackoff.MaxElapsedTime = time.Minute * 2 b := backoff.WithContext(expBackoff, ctx) + stderrR, stderrW := io.Pipe() + stderrScanner := bufio.NewScanner(stderrR) + go func() { + for stderrScanner.Scan() { + e.log.WithField("std", "err").Debug(stderrScanner.Text()) + } + }() + executeSSH := func() error { retCode, err := ssh.Execute( "bastion", - "/bin/true", - []string{}, + []string{"/bin/true"}, + nil, nil, stderrW, ) msg := "error while connecting to bastion host" diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 6efc2a0f85..63561b82aa 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -208,9 +208,9 @@ type Terraform interface { type SSH interface { WriteConfig(Cluster) error - PassThrough([]string) error + PassThrough(host string, additionalArguments []string) error Tunnel(hostname string, destination string, destinationPort int) Tunnel - Execute(host string, cmd string, args []string) (returnCode int, err error) + Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (returnCode int, err error) Validate() error Cleanup() error } diff --git a/pkg/tarmak/ssh.go b/pkg/tarmak/ssh.go index aaaf2eab75..c9b1fedcf1 100644 --- a/pkg/tarmak/ssh.go +++ b/pkg/tarmak/ssh.go @@ -9,16 +9,18 @@ func (t *Tarmak) SSH() interfaces.SSH { return t.ssh } -func (t *Tarmak) SSHPassThrough(argsAdditional []string) { +func (t *Tarmak) SSHPassThrough(host string, argsAdditional []string) error { if err := t.ssh.WriteConfig(t.Cluster()); err != nil { - t.log.Fatal(err) + return err } if err := t.ssh.Validate(); err != nil { - t.log.Fatal(err) + return err } - if err := t.ssh.PassThrough(argsAdditional); err != nil { - t.log.Fatal(err) + if err := t.ssh.PassThrough(host, argsAdditional); err != nil { + return err } + + return nil } diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index 113062fe7c..b61b63ef5e 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -2,18 +2,16 @@ package ssh import ( - //"bufio" "bytes" "encoding/pem" "errors" "fmt" + "io" "io/ioutil" "net" "os" - "os/exec" - // "os/signal" "path/filepath" - "syscall" + "strings" "time" "github.com/hashicorp/go-multierror" @@ -21,6 +19,7 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/terminal" + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/utils" ) @@ -32,6 +31,7 @@ type SSH struct { log *logrus.Entry controlPaths []string + hosts []interfaces.Host } func New(tarmak interfaces.Tarmak) *SSH { @@ -44,11 +44,11 @@ func New(tarmak interfaces.Tarmak) *SSH { } func (s *SSH) WriteConfig(c interfaces.Cluster) error { - hosts, err := c.ListHosts() if err != nil { return err } + s.hosts = hosts var sshConfig bytes.Buffer sshConfig.WriteString(fmt.Sprintf("# ssh config for tarmak cluster %s\n", c.ClusterName())) @@ -75,72 +75,18 @@ func (s *SSH) WriteConfig(c interfaces.Cluster) error { return nil } -func (s *SSH) args() []string { - return []string{ - "ssh", - "-F", - s.tarmak.Cluster().SSHConfigPath(), - } -} - // Pass through a local CLI session -func (s *SSH) PassThrough(argsAdditional []string) error { - hosts, err := s.tarmak.Cluster().ListHosts() - if err != nil { +func (s *SSH) PassThrough(hostName string, argsAdditional []string) error { + if len(argsAdditional) > 0 { + _, err := s.Execute(hostName, argsAdditional, nil, nil, nil) return err } - var host interfaces.Host - var bastion interfaces.Host - for _, h := range hosts { - if h.Aliases()[0] == argsAdditional[0] { - host = h - continue - } - if h.Aliases()[0] == "bastion" { - bastion = h - } - } - - b, err := ioutil.ReadFile(s.tarmak.Environment().SSHPrivateKeyPath()) + client, err := s.client(hostName) if err != nil { return err } - signer, err := ssh.ParsePrivateKey(b) - if err != nil { - return err - } - - conf := &ssh.ClientConfig{ - Timeout: time.Minute * 10, - User: host.User(), - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - confProxy := &ssh.ClientConfig{ - Timeout: time.Minute * 10, - User: host.User(), - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), confProxy) - if err != nil { - return fmt.Errorf("failed to set up connection to bastion: %s", err) - } - conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) - if err != nil { - return fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) - } - - ncc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(host.Hostname(), "22"), conf) - if err != nil { - return fmt.Errorf("failed to set up ssh client: %s", err) - } - - client := ssh.NewClient(ncc, chans, reqs) sess, err := client.NewSession() if err != nil { return err @@ -187,30 +133,45 @@ func (s *SSH) PassThrough(argsAdditional []string) error { return nil } -func (s *SSH) Execute(host string, command string, argsAdditional []string) (returnCode int, err error) { - args := append(s.args(), host, "--", command) - args = append(args, argsAdditional...) - - cmd := exec.Command(args[0], args[1:len(args)]...) +func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + client, err := s.client(host) + if err != nil { + return -1, err + } - err = cmd.Start() + sess, err := client.NewSession() if err != nil { return -1, err } + defer sess.Close() + + if stderr == nil { + sess.Stderr = os.Stderr + } else { + sess.Stderr = stderr + } + + if stdout == nil { + sess.Stdout = os.Stdout + } else { + sess.Stdout = stdout + } + + if stdin == nil { + sess.Stdin = os.Stdin + } else { + sess.Stdin = stdin + } - err = cmd.Wait() + err = sess.Run(strings.Join(cmd, " ")) if err != nil { - perr, ok := err.(*exec.ExitError) - if ok { - if status, ok := perr.Sys().(syscall.WaitStatus); ok { - return status.ExitStatus(), nil - } + if e, ok := err.(*ssh.ExitError); ok { + return e.ExitStatus(), e } return -1, err } return 0, nil - } func (s *SSH) Validate() error { @@ -253,6 +214,83 @@ func (s *SSH) Validate() error { return nil } +func (s *SSH) client(hostName string) (*ssh.Client, error) { + // if we don't have a cache of hosts, write config + if len(s.hosts) == 0 { + err := s.WriteConfig(s.tarmak.Cluster()) + if err != nil { + return nil, err + } + } + + var host interfaces.Host + var bastion interfaces.Host + for _, h := range s.hosts { + for _, a := range h.Aliases() { + if a == hostName { + host = h + } + + if a == clusterv1alpha1.InstancePoolTypeBastion { + bastion = h + } + } + } + + if host == nil || bastion == nil { + return nil, + fmt.Errorf("failed to resolve target hosts for ssh: found %s=%v %s=%v", + clusterv1alpha1.InstancePoolTypeBastion, + !(bastion == nil), hostName, !(host == nil)) + } + + b, err := ioutil.ReadFile(s.tarmak.Environment().SSHPrivateKeyPath()) + if err != nil { + return nil, fmt.Errorf("failed to read ssh private key: %s", err) + } + + signer, err := ssh.ParsePrivateKey(b) + if err != nil { + return nil, fmt.Errorf("failed to parse ssh private key: %s", err) + } + + confProxy := &ssh.ClientConfig{ + Timeout: time.Minute * 10, + User: host.User(), + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), confProxy) + if err != nil { + return nil, fmt.Errorf("failed to set up connection to bastion: %s", err) + } + + // ssh into bastion so no need to set up proxy hop + if hostName == clusterv1alpha1.InstancePoolTypeBastion { + return proxyClient, nil + } + + conf := &ssh.ClientConfig{ + Timeout: time.Minute * 10, + User: host.User(), + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) + if err != nil { + return nil, fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(host.Hostname(), "22"), conf) + if err != nil { + return nil, fmt.Errorf("failed to set up ssh client: %s", err) + } + + return ssh.NewClient(ncc, chans, reqs), nil +} + func (s *SSH) Cleanup() error { var result *multierror.Error diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index f5bb6ab7aa..ea31ae37c5 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -42,6 +42,14 @@ func (s *SSH) Tunnel(hostname string, destination string, destinationPort int) i return t } +func (s *SSH) args() []string { + return []string{ + "ssh", + "-F", + s.tarmak.Cluster().SSHConfigPath(), + } +} + // Start tunnel and wait till a tcp socket is reachable func (t *Tunnel) Start() error { var err error From 9fa953d65d63b904857b4a139cc14ff08ae2c75d Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 20 Nov 2018 18:26:58 +0000 Subject: [PATCH 03/15] Create ssh tunnel with ssh package Signed-off-by: JoshVanL --- pkg/tarmak/interfaces/interfaces.go | 2 +- pkg/tarmak/kubectl/kubectl.go | 32 +++---- pkg/tarmak/ssh/ssh.go | 27 +++--- pkg/tarmak/ssh/tunnel.go | 131 +++++++++++++++------------- pkg/tarmak/vault/tunnel.go | 4 +- pkg/tarmak/vault/vault.go | 5 +- 6 files changed, 106 insertions(+), 95 deletions(-) diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 63561b82aa..5c417c343c 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -217,7 +217,7 @@ type SSH interface { type Tunnel interface { Start() error - Stop() error + Stop() Port() int BindAddress() string } diff --git a/pkg/tarmak/kubectl/kubectl.go b/pkg/tarmak/kubectl/kubectl.go index 3e44adbc95..6963859322 100644 --- a/pkg/tarmak/kubectl/kubectl.go +++ b/pkg/tarmak/kubectl/kubectl.go @@ -21,7 +21,6 @@ import ( clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" - //"github.com/jetstack/tarmak/pkg/tarmak/ssh" "github.com/jetstack/tarmak/pkg/tarmak/utils" ) @@ -30,6 +29,7 @@ var _ interfaces.Kubectl = &Kubectl{} type Kubectl struct { tarmak interfaces.Tarmak log *logrus.Entry + tunnel interfaces.Tunnel } func New(tarmak interfaces.Tarmak) *Kubectl { @@ -200,10 +200,6 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b } } - // If we are using a public endpoint then we don't need to set up a tunnel - // but we need to keep a tunnel var around so we can close is later on if is - // being used. Use k.stopTunnel(tunnel) to ensure no panics. - var tunnel interfaces.Tunnel if publicAPIEndpoint { cluster.Server = fmt.Sprintf("https://api.%s-%s.%s", k.tarmak.Environment().Name(), @@ -211,14 +207,14 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b k.tarmak.Provider().PublicZone()) } else { - tunnel = k.tarmak.Cluster().APITunnel() - if err := tunnel.Start(); err != nil { - k.stopTunnel(tunnel) + k.tunnel = k.tarmak.Cluster().APITunnel() + if err := k.tunnel.Start(); err != nil { + k.stopTunnel() return err } cluster.Server = fmt.Sprintf("https://%s:%d", - tunnel.BindAddress(), tunnel.Port()) + k.tunnel.BindAddress(), k.tunnel.Port()) k.log.Warnf("ssh tunnel connecting to Kubernetes API server will close after 10 minutes: %s", cluster.Server) } @@ -254,9 +250,9 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b } if !publicAPIEndpoint { - k.stopTunnel(tunnel) - tunnel = k.tarmak.Cluster().APITunnel() - err = tunnel.Start() + k.stopTunnel() + k.tunnel = k.tarmak.Cluster().APITunnel() + err = k.tunnel.Start() if err != nil { break } @@ -265,17 +261,17 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b // ensure we close the tunnel on error if err != nil { - k.stopTunnel(tunnel) + k.stopTunnel() return err } if err := utils.EnsureDirectory(filepath.Dir(configPath), 0700); err != nil { - k.stopTunnel(tunnel) + k.stopTunnel() return err } if err := clientcmd.WriteToFile(*c, configPath); err != nil { - k.stopTunnel(tunnel) + k.stopTunnel() return err } @@ -349,8 +345,8 @@ func (k *Kubectl) Kubeconfig(path string, publicAPIEndpoint bool) (string, error return fmt.Sprintf("KUBECONFIG=%s", path), nil } -func (k *Kubectl) stopTunnel(tunnel interfaces.Tunnel) { - if tunnel != nil { - tunnel.Stop() +func (k *Kubectl) stopTunnel() { + if k.tunnel != nil { + k.tunnel.Stop() } } diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index b61b63ef5e..c62e4b850e 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -32,6 +32,7 @@ type SSH struct { controlPaths []string hosts []interfaces.Host + bastion interfaces.Host } func New(tarmak interfaces.Tarmak) *SSH { @@ -86,6 +87,7 @@ func (s *SSH) PassThrough(hostName string, argsAdditional []string) error { if err != nil { return err } + defer client.Close() sess, err := client.NewSession() if err != nil { @@ -138,6 +140,7 @@ func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr if err != nil { return -1, err } + defer client.Close() sess, err := client.NewSession() if err != nil { @@ -163,8 +166,12 @@ func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr sess.Stdin = stdin } - err = sess.Run(strings.Join(cmd, " ")) + err = sess.Start(strings.Join(cmd, " ")) if err != nil { + return -1, err + } + + if err := sess.Wait(); err != nil { if e, ok := err.(*ssh.ExitError); ok { return e.ExitStatus(), e } @@ -233,6 +240,7 @@ func (s *SSH) client(hostName string) (*ssh.Client, error) { if a == clusterv1alpha1.InstancePoolTypeBastion { bastion = h + s.bastion = h } } } @@ -244,6 +252,12 @@ func (s *SSH) client(hostName string) (*ssh.Client, error) { !(bastion == nil), hostName, !(host == nil)) } + if _, err := os.Create(bastion.SSHControlPath()); err != nil { + if !os.IsExist(err) { + return nil, err + } + } + b, err := ioutil.ReadFile(s.tarmak.Environment().SSHPrivateKeyPath()) if err != nil { return nil, fmt.Errorf("failed to read ssh private key: %s", err) @@ -254,14 +268,14 @@ func (s *SSH) client(hostName string) (*ssh.Client, error) { return nil, fmt.Errorf("failed to parse ssh private key: %s", err) } - confProxy := &ssh.ClientConfig{ + conf := &ssh.ClientConfig{ Timeout: time.Minute * 10, User: host.User(), Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), } - proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), confProxy) + proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), conf) if err != nil { return nil, fmt.Errorf("failed to set up connection to bastion: %s", err) } @@ -271,13 +285,6 @@ func (s *SSH) client(hostName string) (*ssh.Client, error) { return proxyClient, nil } - conf := &ssh.ClientConfig{ - Timeout: time.Minute * 10, - User: host.User(), - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) if err != nil { return nil, fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index ea31ae37c5..be46c3d483 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -4,11 +4,12 @@ package ssh import ( "fmt" "io" + "io/ioutil" "net" - "os/exec" "time" "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/utils" @@ -22,31 +23,29 @@ type Tunnel struct { retryCount int retryWait time.Duration + destinaton string + destinatonPort int + forwardSpec string sshCommand []string + ssh *SSH + + serverConn *ssh.Client + listener net.Listener + + remoteConns, localConns []net.Conn } var _ interfaces.Tunnel = &Tunnel{} // This opens a local tunnel through a SSH connection func (s *SSH) Tunnel(hostname string, destination string, destinationPort int) interfaces.Tunnel { - t := &Tunnel{ - localPort: utils.UnusedPort(), - log: s.log.WithField("destination", destination), - retryCount: 30, - retryWait: 500 * time.Millisecond, - sshCommand: s.args(), - } - t.forwardSpec = fmt.Sprintf("-L%s:%d:%s:%d", t.BindAddress(), t.localPort, destination, destinationPort) - - return t -} - -func (s *SSH) args() []string { - return []string{ - "ssh", - "-F", - s.tarmak.Cluster().SSHConfigPath(), + return &Tunnel{ + localPort: utils.UnusedPort(), + log: s.log.WithField("destination", destination), + ssh: s, + destinaton: destination, + destinatonPort: destinationPort, } } @@ -55,73 +54,85 @@ func (t *Tunnel) Start() error { var err error // ensure there is connectivity to the bastion - args := append(t.sshCommand, "bastion", "/bin/true") - cmd := exec.Command(args[0], args[1:len(args)]...) + args := []string{"bastion", "/bin/true"} + t.log.Debugf("checking SSH connection to bastion cmd=%s", args[1]) + ret, err := t.ssh.Execute(args[0], args[1:], nil, nil, nil) + if err != nil || ret != 0 { + return fmt.Errorf("error checking SSH connecting to bastion (%d): %s", ret, err) + } - t.log.Debugf("check SSH connection to bastion cmd=%s", cmd.Args) - err = cmd.Start() + b, err := ioutil.ReadFile(t.ssh.tarmak.Environment().SSHPrivateKeyPath()) if err != nil { - return err + return fmt.Errorf("failed to read ssh private key: %s", err) } - // check for errors - err = cmd.Wait() + signer, err := ssh.ParsePrivateKey(b) if err != nil { - return fmt.Errorf("error checking SSH connecting to bastion: %s", err) + return fmt.Errorf("failed to parse ssh private key: %s", err) } - args = append(t.sshCommand, "-O", "forward", t.forwardSpec, "bastion") - cmd = exec.Command(args[0], args[1:len(args)]...) + confProxy := &ssh.ClientConfig{ + Timeout: time.Minute * 10, + User: t.ssh.bastion.User(), + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } - t.log.Debugf("start tunnel cmd=%s", cmd.Args) - err = cmd.Start() + serverConn, err := ssh.Dial("tcp", net.JoinHostPort(t.ssh.bastion.Hostname(), "22"), confProxy) if err != nil { return err } + t.serverConn = serverConn - // check for errors - err = cmd.Wait() + listener, err := net.Listen("tcp", net.JoinHostPort(t.BindAddress(), fmt.Sprintf("%d", t.Port()))) if err != nil { - return fmt.Errorf("error starting SSH tunnel via bastion: %s", err) + return err } + t.listener = listener + + go t.pass() + + return nil +} - // wait for TCP socket to be reachable - tries := t.retryCount +func (t *Tunnel) pass() { for { - if conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", t.Port()), t.retryWait); err != nil { - t.log.Debug("error connecting to tunnel: ", err) - } else { - conn.Close() - return nil + remoteConn, err := t.serverConn.Dial("tcp", net.JoinHostPort(t.destinaton, fmt.Sprintf("%d", t.destinatonPort))) + if err != nil { + fmt.Errorf("%s\n", err) + return } + t.remoteConns = append(t.remoteConns, remoteConn) - tries -= 1 - if tries == 0 { - break + conn, err := t.listener.Accept() + if err != nil { + t.log.Warnf("error accepting ssh tunnel connection: %s", err) + continue } - time.Sleep(t.retryWait) - } + t.localConns = append(t.localConns, conn) - return fmt.Errorf("could not establish a connection to destination via tunnel after %d tries", t.retryCount) -} - -func (t *Tunnel) Stop() error { - args := append(t.sshCommand, "-O", "cancel", t.forwardSpec, "bastion") - cmd := exec.Command(args[0], args[1:len(args)]...) + go func() { + io.Copy(remoteConn, conn) + remoteConn.Close() + }() - t.log.Debugf("stop tunnel cmd=%s", cmd.Args) - err := cmd.Start() - if err != nil { - return err + go func() { + io.Copy(conn, remoteConn) + conn.Close() + }() } +} - // check for errors - err = cmd.Wait() - if err != nil { - t.log.Warn("stopping ssh tunnel failed with error: ", err) +func (t *Tunnel) Stop() { + for _, l := range t.localConns { + l.Close() + } + for _, r := range t.remoteConns { + r.Close() } - return nil + t.listener.Close() + t.serverConn.Close() } func (t *Tunnel) Port() int { diff --git a/pkg/tarmak/vault/tunnel.go b/pkg/tarmak/vault/tunnel.go index 58d8c000cd..65f2703d63 100644 --- a/pkg/tarmak/vault/tunnel.go +++ b/pkg/tarmak/vault/tunnel.go @@ -64,8 +64,8 @@ func (v *vaultTunnel) Start() error { return nil } -func (v *vaultTunnel) Stop() error { - return v.tunnel.Stop() +func (v *vaultTunnel) Stop() { + v.tunnel.Stop() } func (v *vaultTunnel) Port() int { diff --git a/pkg/tarmak/vault/vault.go b/pkg/tarmak/vault/vault.go index f39a8d799c..ad2b696209 100644 --- a/pkg/tarmak/vault/vault.go +++ b/pkg/tarmak/vault/vault.go @@ -198,10 +198,7 @@ func (v *Vault) VerifyInitFromFQDNs(instances []string, vaultCA, vaultKMSKeyID, wg.Add(1) go func(pos int) { defer wg.Done() - err := tunnels[pos].Stop() - if err != nil { - v.log.Warn(err) - } + tunnels[pos].Stop() }(pos) } wg.Wait() From a1b5d012f98d67172b7569a810b13d78d555430f Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 23 Nov 2018 09:25:25 +0000 Subject: [PATCH 04/15] Uses in package ssh for tunnelling Signed-off-by: JoshVanL --- cmd/tarmak/cmd/tunnel.go | 56 +++++++++++ pkg/tarmak/cluster/cluster.go | 4 +- pkg/tarmak/environment/environment.go | 6 +- pkg/tarmak/interfaces/interfaces.go | 7 +- pkg/tarmak/kubectl/kubectl.go | 4 +- pkg/tarmak/provider/amazon/hosts.go | 6 -- pkg/tarmak/ssh/ssh.go | 128 +++++++++++++----------- pkg/tarmak/ssh/tunnel.go | 138 ++++++++++++++++++-------- pkg/tarmak/tarmak.go | 6 +- pkg/tarmak/vault/tunnel.go | 4 +- pkg/tarmak/vault/vault.go | 2 +- pkg/terraform/terraform.go | 1 + 12 files changed, 237 insertions(+), 125 deletions(-) create mode 100644 cmd/tarmak/cmd/tunnel.go diff --git a/cmd/tarmak/cmd/tunnel.go b/cmd/tarmak/cmd/tunnel.go new file mode 100644 index 0000000000..ac0c2344e8 --- /dev/null +++ b/cmd/tarmak/cmd/tunnel.go @@ -0,0 +1,56 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + "os" + "time" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" +) + +var tunnelCmd = &cobra.Command{ + Use: "tunnel [destination] [port]", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return fmt.Errorf( + "expecting only a destination and port argument, got=%s", args) + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + logrus.Errorf("HERE") + t := tarmak.New(globalFlags) + tunnel := t.SSH().Tunnel(args[0], args[1], false) + + retries := 10 + for { + if retries == 0 { + t.Cleanup() + os.Exit(1) + } + + err := tunnel.Start() + if err == nil { + break + } + + t.Log().Errorf("failed to start tunnel: %s", err) + time.Sleep(time.Second * 2) + retries-- + } + + time.Sleep(time.Minute * 10) + t.Cleanup() + os.Exit(0) + }, + Hidden: true, + DisableFlagParsing: true, +} + +func init() { + RootCmd.AddCommand(tunnelCmd) +} diff --git a/pkg/tarmak/cluster/cluster.go b/pkg/tarmak/cluster/cluster.go index c66347e32d..7002438fb0 100644 --- a/pkg/tarmak/cluster/cluster.go +++ b/pkg/tarmak/cluster/cluster.go @@ -660,9 +660,9 @@ func (c *Cluster) NetworkCIDR() *net.IPNet { func (c *Cluster) APITunnel() interfaces.Tunnel { return c.Environment().Tarmak().SSH().Tunnel( - "bastion", fmt.Sprintf("api.%s.%s", c.ClusterName(), c.Environment().Config().PrivateZone), - 6443, + "6443", + true, ) } diff --git a/pkg/tarmak/environment/environment.go b/pkg/tarmak/environment/environment.go index 3e07a56010..e39c331bc8 100644 --- a/pkg/tarmak/environment/environment.go +++ b/pkg/tarmak/environment/environment.go @@ -297,9 +297,9 @@ func (e *Environment) Verify() error { func (e *Environment) WingTunnel() interfaces.Tunnel { return e.Tarmak().SSH().Tunnel( - "bastion", "localhost", - 9443, + "9443", + false, ) } @@ -311,7 +311,7 @@ func (e *Environment) WingClientset() (*wingclient.Clientset, interfaces.Tunnel, // TODO: Do proper TLS here restConfig := &rest.Config{ - Host: fmt.Sprintf("https://127.0.0.1:%d", tunnel.Port()), + Host: fmt.Sprintf("https://127.0.0.1:%s", tunnel.Port()), TLSClientConfig: rest.TLSClientConfig{ Insecure: true, }, diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 5c417c343c..b27a76cea1 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -209,16 +209,16 @@ type Terraform interface { type SSH interface { WriteConfig(Cluster) error PassThrough(host string, additionalArguments []string) error - Tunnel(hostname string, destination string, destinationPort int) Tunnel + Tunnel(destination, destinationPort string, daemonize bool) Tunnel Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (returnCode int, err error) Validate() error - Cleanup() error + Cleanup() } type Tunnel interface { Start() error Stop() - Port() int + Port() string BindAddress() string } @@ -234,7 +234,6 @@ type Host interface { Roles() []string SSHConfig() string Parameters() map[string]string - SSHControlPath() string Aliases() []string } diff --git a/pkg/tarmak/kubectl/kubectl.go b/pkg/tarmak/kubectl/kubectl.go index 6963859322..09145b4424 100644 --- a/pkg/tarmak/kubectl/kubectl.go +++ b/pkg/tarmak/kubectl/kubectl.go @@ -213,7 +213,7 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b return err } - cluster.Server = fmt.Sprintf("https://%s:%d", + cluster.Server = fmt.Sprintf("https://%s:%s", k.tunnel.BindAddress(), k.tunnel.Port()) k.log.Warnf("ssh tunnel connecting to Kubernetes API server will close after 10 minutes: %s", cluster.Server) @@ -222,7 +222,7 @@ func (k *Kubectl) ensureWorkingKubeconfig(configPath string, publicAPIEndpoint b var err error retries := 5 for { - k.log.Debugf("trying to connect to %+v", cluster.Server) + k.log.Debugf("trying to connect to %s", cluster.Server) var version string version, err = k.verifyAPIVersion(*c) diff --git a/pkg/tarmak/provider/amazon/hosts.go b/pkg/tarmak/provider/amazon/hosts.go index 5391fe76e7..9c3c206e17 100644 --- a/pkg/tarmak/provider/amazon/hosts.go +++ b/pkg/tarmak/provider/amazon/hosts.go @@ -4,7 +4,6 @@ package amazon import ( "fmt" "os" - "path/filepath" "strings" "github.com/aws/aws-sdk-go/aws" @@ -194,8 +193,3 @@ func (a *Amazon) ListHosts(c interfaces.Cluster) ([]interfaces.Host, error) { return hostsInterfaces, nil } - -func (h *host) SSHControlPath() string { - return filepath.Join(os.TempDir(), fmt.Sprintf( - "ssh-control-%s@%s:22", h.user, h.hostname)) -} diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index c62e4b850e..57518b0356 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -14,7 +14,6 @@ import ( "strings" "time" - "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/terminal" @@ -30,9 +29,8 @@ type SSH struct { tarmak interfaces.Tarmak log *logrus.Entry - controlPaths []string - hosts []interfaces.Host - bastion interfaces.Host + hosts map[string]interfaces.Host + tunnels []interfaces.Tunnel } func New(tarmak interfaces.Tarmak) *SSH { @@ -49,18 +47,22 @@ func (s *SSH) WriteConfig(c interfaces.Cluster) error { if err != nil { return err } - s.hosts = hosts var sshConfig bytes.Buffer sshConfig.WriteString(fmt.Sprintf("# ssh config for tarmak cluster %s\n", c.ClusterName())) + s.hosts = make(map[string]interfaces.Host) for _, host := range hosts { _, err = sshConfig.WriteString(host.SSHConfig()) if err != nil { return err } - s.controlPaths = append(s.controlPaths, host.SSHControlPath()) + if len(host.Aliases()) == 0 { + return fmt.Errorf("found host with no aliases: %s", host.Hostname()) + } + + s.hosts[host.Aliases()[0]] = host } err = utils.EnsureDirectory(filepath.Dir(c.SSHConfigPath()), 0700) @@ -222,40 +224,48 @@ func (s *SSH) Validate() error { } func (s *SSH) client(hostName string) (*ssh.Client, error) { - // if we don't have a cache of hosts, write config - if len(s.hosts) == 0 { - err := s.WriteConfig(s.tarmak.Cluster()) - if err != nil { - return nil, err - } + conf, err := s.config() + if err != nil { + return nil, err } - var host interfaces.Host - var bastion interfaces.Host - for _, h := range s.hosts { - for _, a := range h.Aliases() { - if a == hostName { - host = h - } + bastion, err := s.host(clusterv1alpha1.InstancePoolTypeBastion) + if err != nil { + return nil, err + } - if a == clusterv1alpha1.InstancePoolTypeBastion { - bastion = h - s.bastion = h - } - } + proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), conf) + if err != nil { + return nil, fmt.Errorf("failed to set up connection to bastion: %s", err) + } + + // ssh into bastion so no need to set up proxy hop + if hostName == clusterv1alpha1.InstancePoolTypeBastion { + return proxyClient, nil } - if host == nil || bastion == nil { - return nil, - fmt.Errorf("failed to resolve target hosts for ssh: found %s=%v %s=%v", - clusterv1alpha1.InstancePoolTypeBastion, - !(bastion == nil), hostName, !(host == nil)) + host, err := s.host(hostName) + if err != nil { + return nil, err } - if _, err := os.Create(bastion.SSHControlPath()); err != nil { - if !os.IsExist(err) { - return nil, err - } + conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) + if err != nil { + return nil, fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(host.Hostname(), "22"), conf) + if err != nil { + return nil, fmt.Errorf("failed to set up ssh client: %s", err) + } + + return ssh.NewClient(ncc, chans, reqs), nil +} + +func (s *SSH) config() (*ssh.ClientConfig, error) { + bastion, err := s.host(clusterv1alpha1.InstancePoolTypeBastion) + if err != nil { + return nil, err } b, err := ioutil.ReadFile(s.tarmak.Environment().SSHPrivateKeyPath()) @@ -268,44 +278,48 @@ func (s *SSH) client(hostName string) (*ssh.Client, error) { return nil, fmt.Errorf("failed to parse ssh private key: %s", err) } - conf := &ssh.ClientConfig{ + return &ssh.ClientConfig{ Timeout: time.Minute * 10, - User: host.User(), + User: bastion.User(), Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } + }, nil +} - proxyClient, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), conf) - if err != nil { - return nil, fmt.Errorf("failed to set up connection to bastion: %s", err) +func (s *SSH) host(name string) (interfaces.Host, error) { + host, ok := s.hosts[name] + if ok { + return host, nil } - // ssh into bastion so no need to set up proxy hop - if hostName == clusterv1alpha1.InstancePoolTypeBastion { - return proxyClient, nil + // we have already have all hosts, we can't find it + if len(s.hosts) > 0 { + return nil, fmt.Errorf("failed to resolve host: %s", name) } - conn, err := proxyClient.Dial("tcp", net.JoinHostPort(host.Hostname(), "22")) + err := s.WriteConfig(s.tarmak.Cluster()) if err != nil { - return nil, fmt.Errorf("failed to set up connection to %s from basiton: %s", host.Hostname(), err) + return nil, err } - ncc, chans, reqs, err := ssh.NewClientConn(conn, net.JoinHostPort(host.Hostname(), "22"), conf) - if err != nil { - return nil, fmt.Errorf("failed to set up ssh client: %s", err) + _, bok := s.hosts[clusterv1alpha1.InstancePoolTypeBastion] + err = fmt.Errorf("failed to resolve target hosts for ssh: found %s=%v", + clusterv1alpha1.InstancePoolTypeBastion, + bok) + if !bok && name == clusterv1alpha1.InstancePoolTypeBastion { + return nil, err } - return ssh.NewClient(ncc, chans, reqs), nil -} + host, hok := s.hosts[name] + if !hok { + return nil, fmt.Errorf("%s %s=%v", err, name, hok) + } -func (s *SSH) Cleanup() error { - var result *multierror.Error + return host, nil +} - for _, c := range utils.RemoveDuplicateStrings(s.controlPaths) { - if err := os.RemoveAll(c); err != nil { - result = multierror.Append(result, err) - } +func (s *SSH) Cleanup() { + for _, tunnel := range s.tunnels { + tunnel.Stop() } - - return result.ErrorOrNil() } diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index be46c3d483..d1cd859212 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -2,33 +2,32 @@ package ssh import ( + "bufio" "fmt" "io" - "io/ioutil" "net" + "os/exec" + "strconv" + "syscall" "time" + "github.com/kardianos/osext" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/utils" ) type Tunnel struct { - localPort int - log *logrus.Entry - stdin io.WriteCloser + log *logrus.Entry + ssh *SSH - retryCount int - retryWait time.Duration - - destinaton string - destinatonPort int - - forwardSpec string - sshCommand []string - ssh *SSH + dest string + destPort string + localPort string + daemonize bool serverConn *ssh.Client listener net.Listener @@ -39,20 +38,22 @@ type Tunnel struct { var _ interfaces.Tunnel = &Tunnel{} // This opens a local tunnel through a SSH connection -func (s *SSH) Tunnel(hostname string, destination string, destinationPort int) interfaces.Tunnel { - return &Tunnel{ - localPort: utils.UnusedPort(), - log: s.log.WithField("destination", destination), - ssh: s, - destinaton: destination, - destinatonPort: destinationPort, +func (s *SSH) Tunnel(dest, destPort string, daemonize bool) interfaces.Tunnel { + tunnel := &Tunnel{ + log: s.log.WithField("destination", dest), + ssh: s, + dest: dest, + destPort: destPort, + daemonize: daemonize, + localPort: strconv.Itoa(utils.UnusedPort()), } + + s.tunnels = append(s.tunnels, tunnel) + return tunnel } // Start tunnel and wait till a tcp socket is reachable func (t *Tunnel) Start() error { - var err error - // ensure there is connectivity to the bastion args := []string{"bastion", "/bin/true"} t.log.Debugf("checking SSH connection to bastion cmd=%s", args[1]) @@ -60,47 +61,48 @@ func (t *Tunnel) Start() error { if err != nil || ret != 0 { return fmt.Errorf("error checking SSH connecting to bastion (%d): %s", ret, err) } + t.log.Debug("connection to bastion successful") - b, err := ioutil.ReadFile(t.ssh.tarmak.Environment().SSHPrivateKeyPath()) - if err != nil { - return fmt.Errorf("failed to read ssh private key: %s", err) + if t.daemonize { + return t.startDaemon() } - signer, err := ssh.ParsePrivateKey(b) + conf, err := t.ssh.config() if err != nil { - return fmt.Errorf("failed to parse ssh private key: %s", err) + return err } - confProxy := &ssh.ClientConfig{ - Timeout: time.Minute * 10, - User: t.ssh.bastion.User(), - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + bastion, err := t.ssh.host(clusterv1alpha1.InstancePoolTypeBastion) + if err != nil { + return err } - serverConn, err := ssh.Dial("tcp", net.JoinHostPort(t.ssh.bastion.Hostname(), "22"), confProxy) + serverConn, err := ssh.Dial("tcp", net.JoinHostPort(bastion.Hostname(), "22"), conf) if err != nil { return err } t.serverConn = serverConn - listener, err := net.Listen("tcp", net.JoinHostPort(t.BindAddress(), fmt.Sprintf("%d", t.Port()))) + listener, err := net.Listen("tcp", net.JoinHostPort(t.BindAddress(), t.Port())) if err != nil { return err } t.listener = listener - go t.pass() + go t.handle() return nil } -func (t *Tunnel) pass() { +func (t *Tunnel) handle() { for { - remoteConn, err := t.serverConn.Dial("tcp", net.JoinHostPort(t.destinaton, fmt.Sprintf("%d", t.destinatonPort))) + remoteConn, err := t.serverConn.Dial("tcp", + net.JoinHostPort(t.dest, t.destPort)) if err != nil { - fmt.Errorf("%s\n", err) - return + net.ErrWriteToConnected. + t.log.Errorf("failed to create tunnel: %s", err) + time.Sleep(time.Second * 2) + continue } t.remoteConns = append(t.remoteConns, remoteConn) @@ -125,20 +127,68 @@ func (t *Tunnel) pass() { func (t *Tunnel) Stop() { for _, l := range t.localConns { - l.Close() + if l != nil { + l.Close() + } } for _, r := range t.remoteConns { - r.Close() + if r != nil { + r.Close() + } } - t.listener.Close() - t.serverConn.Close() + if t.listener != nil { + t.listener.Close() + } + if t.serverConn != nil { + t.serverConn.Close() + } } -func (t *Tunnel) Port() int { +func (t *Tunnel) Port() string { return t.localPort } func (t *Tunnel) BindAddress() string { return "127.0.0.1" } + +func (t *Tunnel) startDaemon() error { + binaryPath, err := osext.Executable() + if err != nil { + return fmt.Errorf("error finding tarmak executable: %s", err) + } + + cmd := exec.Command(binaryPath, "tunnel", t.dest, t.destPort) + + outR, outW := io.Pipe() + errR, errW := io.Pipe() + outS := bufio.NewScanner(outR) + errS := bufio.NewScanner(errR) + + cmd.Stdin = nil + cmd.Stdout = outW + cmd.Stderr = errW + + go func() { + for outS.Scan() { + t.log.WithField("tunnel", t.dest).Info(outS.Text()) + } + }() + go func() { + for errS.Scan() { + t.log.WithField("tunnel", t.dest).Error(errS.Text()) + } + }() + + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + Foreground: false, + } + + if err := cmd.Start(); err != nil { + return err + } + + return nil +} diff --git a/pkg/tarmak/tarmak.go b/pkg/tarmak/tarmak.go index e0faa5585c..3a092f75dd 100644 --- a/pkg/tarmak/tarmak.go +++ b/pkg/tarmak/tarmak.go @@ -361,13 +361,11 @@ func (t *Tarmak) Cleanup() { t.rootPath = nil } - if err := t.SSH().Cleanup(); err != nil { - t.log.Warnf("error cleaning up ssh run time assets: %s", err) - } - if err := t.terraform.Cleanup(); err != nil { t.log.Warnf("error cleaning up terraform run time assets: %s", err) } + + t.ssh.Cleanup() } func (t *Tarmak) Variables() map[string]interface{} { diff --git a/pkg/tarmak/vault/tunnel.go b/pkg/tarmak/vault/tunnel.go index 65f2703d63..4b9ac2fc3a 100644 --- a/pkg/tarmak/vault/tunnel.go +++ b/pkg/tarmak/vault/tunnel.go @@ -38,7 +38,7 @@ func NewTunnel( } err = vaultClient.SetAddress( fmt.Sprintf( - "https://%s:%d", tunnel.BindAddress(), tunnel.Port(), + "https://%s:%s", tunnel.BindAddress(), tunnel.Port(), ), ) if err != nil { @@ -68,7 +68,7 @@ func (v *vaultTunnel) Stop() { v.tunnel.Stop() } -func (v *vaultTunnel) Port() int { +func (v *vaultTunnel) Port() string { return v.tunnel.Port() } diff --git a/pkg/tarmak/vault/vault.go b/pkg/tarmak/vault/vault.go index ad2b696209..2c37dbefee 100644 --- a/pkg/tarmak/vault/vault.go +++ b/pkg/tarmak/vault/vault.go @@ -154,7 +154,7 @@ func (v *Vault) createTunnelsWithCA(instances []string, vaultCA string) ([]*vaul for pos := range instances { fqdn := instances[pos] sshTunnel := v.cluster.Environment().Tarmak().SSH().Tunnel( - "bastion", fqdn, 8200, + fqdn, "8200", false, ) vaultTunnel, err := NewTunnel( sshTunnel, diff --git a/pkg/terraform/terraform.go b/pkg/terraform/terraform.go index 5cee4dc908..224b706da3 100644 --- a/pkg/terraform/terraform.go +++ b/pkg/terraform/terraform.go @@ -60,6 +60,7 @@ func New(tarmak interfaces.Tarmak) *Terraform { // terraform providers and provisioners in general. We are pointing through // symlinks to the tarmak binary, which contains all relevant providers func (t *Terraform) preparePlugins(c interfaces.Cluster) error { + osext.Executable() binaryPath, err := osext.Executable() if err != nil { return fmt.Errorf("error finding tarmak executable: %s", err) From 410617ffc51e096190e9987956260f32888445ab Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 23 Nov 2018 12:10:46 +0000 Subject: [PATCH 05/15] Use same local port when daemoning tunnel Signed-off-by: JoshVanL --- cmd/tarmak/cmd/tunnel.go | 11 +++---- pkg/tarmak/cluster/cluster.go | 3 ++ pkg/tarmak/environment/environment.go | 2 ++ pkg/tarmak/interfaces/interfaces.go | 2 +- pkg/tarmak/ssh/tunnel.go | 44 ++++++++++++++++++--------- pkg/tarmak/vault/vault.go | 3 +- 6 files changed, 43 insertions(+), 22 deletions(-) diff --git a/cmd/tarmak/cmd/tunnel.go b/cmd/tarmak/cmd/tunnel.go index ac0c2344e8..64de2f71d4 100644 --- a/cmd/tarmak/cmd/tunnel.go +++ b/cmd/tarmak/cmd/tunnel.go @@ -6,25 +6,23 @@ import ( "os" "time" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/jetstack/tarmak/pkg/tarmak" ) var tunnelCmd = &cobra.Command{ - Use: "tunnel [destination] [port]", + Use: "tunnel [destination] [destination port] [local port]", PreRunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 2 { + if len(args) != 3 { return fmt.Errorf( - "expecting only a destination and port argument, got=%s", args) + "expecting only a destination, destination and local port argument, got=%s", args) } return nil }, Run: func(cmd *cobra.Command, args []string) { - logrus.Errorf("HERE") t := tarmak.New(globalFlags) - tunnel := t.SSH().Tunnel(args[0], args[1], false) + tunnel := t.SSH().Tunnel(args[0], args[1], args[2], false) retries := 10 for { @@ -35,6 +33,7 @@ var tunnelCmd = &cobra.Command{ err := tunnel.Start() if err == nil { + t.Log().Infof("tunnel started: %s", args) break } diff --git a/pkg/tarmak/cluster/cluster.go b/pkg/tarmak/cluster/cluster.go index 7002438fb0..473fa928f1 100644 --- a/pkg/tarmak/cluster/cluster.go +++ b/pkg/tarmak/cluster/cluster.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "path/filepath" + "strconv" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" @@ -16,6 +17,7 @@ import ( "github.com/jetstack/tarmak/pkg/tarmak/instance_pool" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/role" + "github.com/jetstack/tarmak/pkg/tarmak/utils" wingclient "github.com/jetstack/tarmak/pkg/wing/client/clientset/versioned" ) @@ -662,6 +664,7 @@ func (c *Cluster) APITunnel() interfaces.Tunnel { return c.Environment().Tarmak().SSH().Tunnel( fmt.Sprintf("api.%s.%s", c.ClusterName(), c.Environment().Config().PrivateZone), "6443", + strconv.Itoa(utils.UnusedPort()), true, ) } diff --git a/pkg/tarmak/environment/environment.go b/pkg/tarmak/environment/environment.go index e39c331bc8..54327ab6cb 100644 --- a/pkg/tarmak/environment/environment.go +++ b/pkg/tarmak/environment/environment.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" @@ -299,6 +300,7 @@ func (e *Environment) WingTunnel() interfaces.Tunnel { return e.Tarmak().SSH().Tunnel( "localhost", "9443", + strconv.Itoa(utils.UnusedPort()), false, ) } diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index b27a76cea1..3388673962 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -209,7 +209,7 @@ type Terraform interface { type SSH interface { WriteConfig(Cluster) error PassThrough(host string, additionalArguments []string) error - Tunnel(destination, destinationPort string, daemonize bool) Tunnel + Tunnel(destination, destinationPort, localPort string, daemonize bool) Tunnel Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (returnCode int, err error) Validate() error Cleanup() diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index d1cd859212..67bf8327d9 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -7,7 +7,6 @@ import ( "io" "net" "os/exec" - "strconv" "syscall" "time" @@ -17,12 +16,12 @@ import ( clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" - "github.com/jetstack/tarmak/pkg/tarmak/utils" ) type Tunnel struct { - log *logrus.Entry - ssh *SSH + log *logrus.Entry + ssh *SSH + stopCh chan struct{} dest string destPort string @@ -38,14 +37,15 @@ type Tunnel struct { var _ interfaces.Tunnel = &Tunnel{} // This opens a local tunnel through a SSH connection -func (s *SSH) Tunnel(dest, destPort string, daemonize bool) interfaces.Tunnel { +func (s *SSH) Tunnel(dest, destPort, localPort string, daemonize bool) interfaces.Tunnel { tunnel := &Tunnel{ log: s.log.WithField("destination", dest), ssh: s, dest: dest, destPort: destPort, daemonize: daemonize, - localPort: strconv.Itoa(utils.UnusedPort()), + localPort: localPort, + stopCh: make(chan struct{}), } s.tunnels = append(s.tunnels, tunnel) @@ -64,7 +64,13 @@ func (t *Tunnel) Start() error { t.log.Debug("connection to bastion successful") if t.daemonize { - return t.startDaemon() + err := t.startDaemon() + if err != nil { + return err + } + + time.Sleep(time.Second * 2) + return nil } conf, err := t.ssh.config() @@ -99,15 +105,19 @@ func (t *Tunnel) handle() { remoteConn, err := t.serverConn.Dial("tcp", net.JoinHostPort(t.dest, t.destPort)) if err != nil { - net.ErrWriteToConnected. - t.log.Errorf("failed to create tunnel: %s", err) - time.Sleep(time.Second * 2) - continue + t.log.Errorf("failed to create tunnel: %s", err) + return } t.remoteConns = append(t.remoteConns, remoteConn) conn, err := t.listener.Accept() if err != nil { + select { + case <-t.stopCh: + return + default: + } + t.log.Warnf("error accepting ssh tunnel connection: %s", err) continue } @@ -126,6 +136,12 @@ func (t *Tunnel) handle() { } func (t *Tunnel) Stop() { + select { + case <-t.stopCh: + default: + close(t.stopCh) + } + for _, l := range t.localConns { if l != nil { l.Close() @@ -159,7 +175,7 @@ func (t *Tunnel) startDaemon() error { return fmt.Errorf("error finding tarmak executable: %s", err) } - cmd := exec.Command(binaryPath, "tunnel", t.dest, t.destPort) + cmd := exec.Command(binaryPath, "tunnel", t.dest, t.destPort, t.localPort) outR, outW := io.Pipe() errR, errW := io.Pipe() @@ -172,12 +188,12 @@ func (t *Tunnel) startDaemon() error { go func() { for outS.Scan() { - t.log.WithField("tunnel", t.dest).Info(outS.Text()) + t.log.WithField("tunnel", t.dest).Debug(outS.Text()) } }() go func() { for errS.Scan() { - t.log.WithField("tunnel", t.dest).Error(errS.Text()) + t.log.WithField("tunnel", t.dest).Debug(errS.Text()) } }() diff --git a/pkg/tarmak/vault/vault.go b/pkg/tarmak/vault/vault.go index 2c37dbefee..910a26a727 100644 --- a/pkg/tarmak/vault/vault.go +++ b/pkg/tarmak/vault/vault.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "sync" "time" @@ -154,7 +155,7 @@ func (v *Vault) createTunnelsWithCA(instances []string, vaultCA string) ([]*vaul for pos := range instances { fqdn := instances[pos] sshTunnel := v.cluster.Environment().Tarmak().SSH().Tunnel( - fqdn, "8200", false, + fqdn, "8200", strconv.Itoa(utils.UnusedPort()), false, ) vaultTunnel, err := NewTunnel( sshTunnel, From e2f597f6b3b8efca14475a7f4cea4427d902d605 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 23 Nov 2018 13:02:25 +0000 Subject: [PATCH 06/15] Set tunnel tries to 5 and exit immediately Signed-off-by: JoshVanL --- cmd/tarmak/cmd/tunnel.go | 15 ++++++++------- pkg/tarmak/ssh/tunnel.go | 6 ++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/tarmak/cmd/tunnel.go b/cmd/tarmak/cmd/tunnel.go index 64de2f71d4..d0e8d5eafc 100644 --- a/cmd/tarmak/cmd/tunnel.go +++ b/cmd/tarmak/cmd/tunnel.go @@ -24,13 +24,8 @@ var tunnelCmd = &cobra.Command{ t := tarmak.New(globalFlags) tunnel := t.SSH().Tunnel(args[0], args[1], args[2], false) - retries := 10 + retries := 5 for { - if retries == 0 { - t.Cleanup() - os.Exit(1) - } - err := tunnel.Start() if err == nil { t.Log().Infof("tunnel started: %s", args) @@ -38,8 +33,14 @@ var tunnelCmd = &cobra.Command{ } t.Log().Errorf("failed to start tunnel: %s", err) - time.Sleep(time.Second * 2) retries-- + if retries == 0 { + t.Log().Error("failed to start tunnel after 5 attempts") + t.Cleanup() + os.Exit(1) + } + + time.Sleep(time.Second * 2) } time.Sleep(time.Minute * 10) diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index 67bf8327d9..8084e334f1 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -55,13 +55,10 @@ func (s *SSH) Tunnel(dest, destPort, localPort string, daemonize bool) interface // Start tunnel and wait till a tcp socket is reachable func (t *Tunnel) Start() error { // ensure there is connectivity to the bastion - args := []string{"bastion", "/bin/true"} - t.log.Debugf("checking SSH connection to bastion cmd=%s", args[1]) - ret, err := t.ssh.Execute(args[0], args[1:], nil, nil, nil) + ret, err := t.ssh.Execute("bastion", []string{"/bin/true"}, nil, nil, nil) if err != nil || ret != 0 { return fmt.Errorf("error checking SSH connecting to bastion (%d): %s", ret, err) } - t.log.Debug("connection to bastion successful") if t.daemonize { err := t.startDaemon() @@ -69,6 +66,7 @@ func (t *Tunnel) Start() error { return err } + // allow for some warm up time time.Sleep(time.Second * 2) return nil } From b98cdb769f9cf6bcccf05c26ba230d3624ee0623 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 23 Nov 2018 13:05:18 +0000 Subject: [PATCH 07/15] Update vault tunnel test Signed-off-by: JoshVanL --- pkg/tarmak/vault/tunnel_test.go | 15 +++++---------- pkg/terraform/terraform.go | 1 - 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/tarmak/vault/tunnel_test.go b/pkg/tarmak/vault/tunnel_test.go index f4a4d04721..54775f2a28 100644 --- a/pkg/tarmak/vault/tunnel_test.go +++ b/pkg/tarmak/vault/tunnel_test.go @@ -7,7 +7,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "strconv" "testing" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" @@ -16,10 +15,10 @@ import ( type FakeTunnel struct { bindAddress string - port int + port string } -func (ft *FakeTunnel) Port() int { +func (ft *FakeTunnel) Port() string { return ft.port } @@ -31,8 +30,8 @@ func (ft *FakeTunnel) Start() error { return nil } -func (ft *FakeTunnel) Stop() error { - return nil +func (ft *FakeTunnel) Stop() { + return } var _ interfaces.Tunnel = &FakeTunnel{} @@ -60,13 +59,9 @@ func TestVaultTunnel(t *testing.T) { if err != nil { t.Fatal(err) } - port, err := strconv.Atoi(u.Port()) - if err != nil { - t.Fatal(err) - } tunnel := &FakeTunnel{ bindAddress: u.Hostname(), - port: port, + port: u.Port(), } fqdn := "host1.example.com" vaultCA := x509.NewCertPool() diff --git a/pkg/terraform/terraform.go b/pkg/terraform/terraform.go index 224b706da3..5cee4dc908 100644 --- a/pkg/terraform/terraform.go +++ b/pkg/terraform/terraform.go @@ -60,7 +60,6 @@ func New(tarmak interfaces.Tarmak) *Terraform { // terraform providers and provisioners in general. We are pointing through // symlinks to the tarmak binary, which contains all relevant providers func (t *Terraform) preparePlugins(c interfaces.Cluster) error { - osext.Executable() binaryPath, err := osext.Executable() if err != nil { return fmt.Errorf("error finding tarmak executable: %s", err) From 3d389be3f9650e4d8ae7688a71169e9aa2fdd82b Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 23 Nov 2018 13:15:43 +0000 Subject: [PATCH 08/15] Update Gopkg.lock and cmd docs Signed-off-by: JoshVanL --- Gopkg.lock | 1 + docs/cmd-docs.rst | 5 --- .../tarmak/tarmak_clusters_instances_ssh.rst | 42 ------------------- .../cmd/tarmak/tarmak_clusters_ssh.rst | 2 +- 4 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_instances_ssh.rst diff --git a/Gopkg.lock b/Gopkg.lock index bb895973e0..0252481b14 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2687,6 +2687,7 @@ "github.com/terraform-providers/terraform-provider-template/template", "github.com/terraform-providers/terraform-provider-tls/tls", "golang.org/x/crypto/ssh", + "golang.org/x/crypto/ssh/terminal", "golang.org/x/net/context", "gopkg.in/src-d/go-git.v4", "gopkg.in/src-d/go-git.v4/config", diff --git a/docs/cmd-docs.rst b/docs/cmd-docs.rst index bd1d9ef050..ec1582433f 100644 --- a/docs/cmd-docs.rst +++ b/docs/cmd-docs.rst @@ -79,11 +79,6 @@ Command line documentation for both tarmak and wing commands generated/cmd/tarmak/tarmak_clusters_instances_list -.. toctree:: - :maxdepth: 1 - - generated/cmd/tarmak/tarmak_clusters_instances_ssh - .. toctree:: :maxdepth: 1 diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_instances_ssh.rst b/docs/generated/cmd/tarmak/tarmak_clusters_instances_ssh.rst deleted file mode 100644 index 017820f6cf..0000000000 --- a/docs/generated/cmd/tarmak/tarmak_clusters_instances_ssh.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. _tarmak_clusters_instances_ssh: - -tarmak clusters instances ssh ------------------------------ - -Log into an instance with SSH - -Synopsis -~~~~~~~~ - - -Log into an instance with SSH - -:: - - tarmak clusters instances ssh [instance alias] [flags] - -Options -~~~~~~~ - -:: - - -h, --help help for ssh - -Options inherited from parent commands -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") - --current-cluster string override the current cluster set in the config - --keep-containers do not clean-up terraform/packer containers after running them - --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) - --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint - -v, --verbose enable verbose logging - --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub - -SEE ALSO -~~~~~~~~ - -* `tarmak clusters instances `_ - Operations on instances - diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_ssh.rst b/docs/generated/cmd/tarmak/tarmak_clusters_ssh.rst index a0a1525712..0208b5f168 100644 --- a/docs/generated/cmd/tarmak/tarmak_clusters_ssh.rst +++ b/docs/generated/cmd/tarmak/tarmak_clusters_ssh.rst @@ -13,7 +13,7 @@ Log into an instance with SSH :: - tarmak clusters ssh [instance alias] [flags] + tarmak clusters ssh [instance alias] [optional ssh arguments] [flags] Options ~~~~~~~ From f981b6101925c9e055f7ff22ad0dc0454634b4bc Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Mon, 5 Nov 2018 14:39:39 +0000 Subject: [PATCH 09/15] Adds snapshot save and restore for consul Signed-off-by: JoshVanL --- cmd/tarmak/cmd/cluster_snapshot.go | 15 ++ cmd/tarmak/cmd/cluster_snapshot_consul.go | 15 ++ .../cmd/cluster_snapshot_consul_restore.go | 32 +++++ .../cmd/cluster_snapshot_consul_save.go | 32 +++++ cmd/tarmak/cmd/cluster_snapshot_etcd.go | 15 ++ cmd/tarmak/cmd/cluster_snapshot_etcd_save.go | 17 +++ pkg/tarmak/cmd.go | 20 +++ pkg/tarmak/interfaces/interfaces.go | 5 + pkg/tarmak/snapshot/consul/consul.go | 136 ++++++++++++++++++ pkg/tarmak/snapshot/snapshot.go | 48 +++++++ pkg/tarmak/ssh/ssh.go | 1 + pkg/tarmak/utils/context.go | 2 +- pkg/tarmak/utils/slices.go | 13 ++ 13 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 cmd/tarmak/cmd/cluster_snapshot.go create mode 100644 cmd/tarmak/cmd/cluster_snapshot_consul.go create mode 100644 cmd/tarmak/cmd/cluster_snapshot_consul_restore.go create mode 100644 cmd/tarmak/cmd/cluster_snapshot_consul_save.go create mode 100644 cmd/tarmak/cmd/cluster_snapshot_etcd.go create mode 100644 cmd/tarmak/cmd/cluster_snapshot_etcd_save.go create mode 100644 pkg/tarmak/snapshot/consul/consul.go create mode 100644 pkg/tarmak/snapshot/snapshot.go diff --git a/cmd/tarmak/cmd/cluster_snapshot.go b/cmd/tarmak/cmd/cluster_snapshot.go new file mode 100644 index 0000000000..d1faa1aefe --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "Manage snapshots of remote consul and etcd clusters", +} + +func init() { + clusterCmd.AddCommand(clusterSnapshotCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul.go b/cmd/tarmak/cmd/cluster_snapshot_consul.go new file mode 100644 index 0000000000..13b067c926 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotConsulCmd = &cobra.Command{ + Use: "consul", + Short: "Manage snapshots on remote consul clusters", +} + +func init() { + clusterSnapshotCmd.AddCommand(clusterSnapshotConsulCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go new file mode 100644 index 0000000000..6f0939f33c --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go @@ -0,0 +1,32 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/consul" +) + +var clusterSnapshotConsulRestoreCmd = &cobra.Command{ + Use: "restore [source path]", + Short: "restore consul cluster with source snapshot", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single source path, got=%d", len(args)) + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + c := consul.NewConsul(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, c).Restore) + }, +} + +func init() { + clusterSnapshotConsulCmd.AddCommand(clusterSnapshotConsulRestoreCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_save.go b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go new file mode 100644 index 0000000000..9718abab5c --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go @@ -0,0 +1,32 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/consul" +) + +var clusterSnapshotConsulSaveCmd = &cobra.Command{ + Use: "save [target path]", + Short: "save consul cluster snapshot to target path", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single target path, got=%d", len(args)) + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + c := consul.NewConsul(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, c).Save) + }, +} + +func init() { + clusterSnapshotConsulCmd.AddCommand(clusterSnapshotConsulSaveCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd.go b/cmd/tarmak/cmd/cluster_snapshot_etcd.go new file mode 100644 index 0000000000..f14853cb62 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotEtcdCmd = &cobra.Command{ + Use: "etcd", + Short: "Manage snapshots on remote etcd clusters", +} + +func init() { + clusterSnapshotCmd.AddCommand(clusterSnapshotEtcdCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go new file mode 100644 index 0000000000..3438142e35 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go @@ -0,0 +1,17 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotEtcdSaveCmd = &cobra.Command{ + Use: "save [target path]", + Short: "save etcd snapshot to target path", + Run: func(cmd *cobra.Command, args []string) { + }, +} + +func init() { + clusterSnapshotEtcdCmd.AddCommand(clusterSnapshotEtcdSaveCmd) +} diff --git a/pkg/tarmak/cmd.go b/pkg/tarmak/cmd.go index 78d6399324..19ba4d03ff 100644 --- a/pkg/tarmak/cmd.go +++ b/pkg/tarmak/cmd.go @@ -30,6 +30,11 @@ type CmdTarmak struct { ctx interfaces.CancellationContext } +type CmdSnapshot struct { + *CmdTarmak + snapshot interfaces.Snapshot +} + func (t *Tarmak) NewCmdTarmak(pflags *pflag.FlagSet, args []string) *CmdTarmak { return &CmdTarmak{ Tarmak: t, @@ -40,6 +45,13 @@ func (t *Tarmak) NewCmdTarmak(pflags *pflag.FlagSet, args []string) *CmdTarmak { } } +func (t *Tarmak) NewCmdSnapshot(pflags *pflag.FlagSet, args []string, sh interfaces.Snapshot) *CmdSnapshot { + return &CmdSnapshot{ + CmdTarmak: t.NewCmdTarmak(pflags, args), + snapshot: sh, + } +} + func (c *CmdTarmak) Plan() (returnCode int, err error) { if err := c.setup(); err != nil { return 1, err @@ -286,6 +298,14 @@ func (c *CmdTarmak) kubePublicAPIEndpoint() bool { return publicEndpoint } +func (c *CmdSnapshot) Save() error { + return c.snapshot.Save() +} + +func (c *CmdSnapshot) Restore() error { + return c.snapshot.Restore() +} + func (c *CmdTarmak) verifyTerraformBinaryVersion() error { cmd := exec.Command("terraform", "version") cmd.Env = os.Environ() diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 3388673962..db94d6c6b1 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -292,3 +292,8 @@ type CancellationContext interface { WaitOrCancel(f func() error) WaitOrCancelReturnCode(f func() (int, error)) } + +type Snapshot interface { + Save() error + Restore() error +} diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go new file mode 100644 index 0000000000..343a28fbc1 --- /dev/null +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -0,0 +1,136 @@ +package consul + +import ( + "bufio" + "fmt" + "io" + "strings" + "time" + + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot" +) + +var _ interfaces.Snapshot = &Consul{} + +const ( + snapshotTimeLayout = "2006-01-02_15-04-05" +) + +var ( + exportCmd = []string{ + "export", + "CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token');", + } +) + +type Consul struct { + tarmak interfaces.Tarmak + ssh interfaces.SSH + log *logrus.Entry + + path string + aliases []string +} + +func NewConsul(tarmak interfaces.Tarmak, path string) *Consul { + return &Consul{ + tarmak: tarmak, + ssh: tarmak.SSH(), + log: tarmak.Log(), + path: path, + } +} + +func (c *Consul) Save() error { + aliases, err := snapshot.Prepare(c.tarmak, clusterv1alpha1.InstancePoolTypeVault) + if err != nil { + return err + } + c.aliases = aliases + + c.log.Infof("saving snapshot from instance %s", aliases[0]) + targetPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshotTimeLayout)) + + cmdArgs := append(exportCmd, "consul", "snapshot", "save", targetPath) + err = c.sshCmd( + aliases[0], + cmdArgs[0], + cmdArgs[1:], + ) + if err != nil { + return err + } + + ret, err := c.ssh.ScpToLocal(aliases[0], targetPath, c.path) + if ret != 0 { + cmdStr := fmt.Sprintf("%s", strings.Join(cmdArgs, " ")) + return fmt.Errorf("command [%s] returned non-zero: %d, %s", cmdStr, ret, err) + } + + c.log.Infof("consul snapshot saved to %s", c.path) + + return err +} + +func (c *Consul) Restore() error { + aliases, err := snapshot.Prepare(c.tarmak, clusterv1alpha1.InstancePoolTypeVault) + if err != nil { + return err + } + c.aliases = aliases + + for _, a := range aliases { + c.log.Infof("restoring snapshot to instance %s", a) + targetPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshotTimeLayout)) + + ret, err := c.ssh.ScpToHost(a, c.path, targetPath) + if ret != 0 { + return fmt.Errorf("command scp returned non-zero: %d, %s", ret, err) + } + + cmdArgs := append(exportCmd, "consul", "snapshot", "restore", targetPath) + err = c.sshCmd( + a, + cmdArgs[0], + cmdArgs[1:], + ) + if err != nil { + return err + } + } + + c.log.Infof("consul snapshot restored from %s", c.path) + + return err +} + +func (c *Consul) sshCmd(host, command string, args []string) error { + readerO, writerO := io.Pipe() + readerE, writerE := io.Pipe() + scannerO := bufio.NewScanner(readerO) + scannerE := bufio.NewScanner(readerE) + + go func() { + for scannerO.Scan() { + c.log.WithField("std", "out").Debug(scannerO.Text()) + } + }() + + go func() { + for scannerE.Scan() { + c.log.WithField("std", "err").Warn(scannerE.Text()) + } + }() + + ret, err := c.ssh.ExecuteWithWriter(host, command, args, writerO, writerE) + if ret != 0 { + cmdStr := fmt.Sprintf("%s %s", command, strings.Join(args, " ")) + return fmt.Errorf("command [%s] returned non-zero: %d", cmdStr, ret) + } + + return err +} diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go new file mode 100644 index 0000000000..c78e3fd809 --- /dev/null +++ b/pkg/tarmak/snapshot/snapshot.go @@ -0,0 +1,48 @@ +package snapshot + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/utils" +) + +func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error) { + if err := tarmak.SSH().WriteConfig(tarmak.Cluster()); err != nil { + return nil, err + } + + hosts, err := tarmak.Cluster().ListHosts() + if err != nil { + return nil, err + } + + var result *multierror.Error + for _, host := range hosts { + if utils.SliceContainsPrefix(host.Roles(), role) { + if len(host.Aliases()) == 0 { + err := fmt.Errorf( + "host with correct role '%v' found without alias: %v", + host.Roles(), + host.ID(), + ) + result = multierror.Append(result, err) + continue + } + + aliases = append(aliases, host.Aliases()[0]) + } + } + + if result != nil { + return nil, result + } + + if len(aliases) == 0 { + return nil, fmt.Errorf("no host aliases were found with the role %s", role) + } + + return aliases, result.ErrorOrNil() +} diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index 57518b0356..1316cb98d6 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -177,6 +177,7 @@ func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr if e, ok := err.(*ssh.ExitError); ok { return e.ExitStatus(), e } + return -1, err } diff --git a/pkg/tarmak/utils/context.go b/pkg/tarmak/utils/context.go index 8be5a9f9eb..968a315542 100644 --- a/pkg/tarmak/utils/context.go +++ b/pkg/tarmak/utils/context.go @@ -110,7 +110,7 @@ func (c *CancellationContext) WaitOrCancelReturnCode(f func() (int, error)) { wg.Done() return case <-time.After(time.Second * 3): - log.Warn("tarmak is shutting down") + log.Warn("tarmak is shutting down...") log.Warn("* tarmak will attempt to kill the current task") log.Warn("* send another SIGTERM or SIGINT (ctrl-c) to exit immediately") } diff --git a/pkg/tarmak/utils/slices.go b/pkg/tarmak/utils/slices.go index 292a1c2bee..938979232e 100644 --- a/pkg/tarmak/utils/slices.go +++ b/pkg/tarmak/utils/slices.go @@ -1,6 +1,10 @@ // Copyright Jetstack Ltd. See LICENSE for details. package utils +import ( + "strings" +) + func RemoveDuplicateStrings(slice []string) (result []string) { seen := make(map[string]bool) @@ -36,3 +40,12 @@ func SliceContains(slice []string, str string) bool { return false } + +func SliceContainsPrefix(slice []string, prefix string) bool { + for _, s := range slice { + if strings.HasPrefix(s, prefix) { + return true + } + } + return false +} From 2551d5e0ac760d291b1820241f0ddb7551e3f092 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Wed, 7 Nov 2018 15:02:41 +0000 Subject: [PATCH 10/15] Adds tarmak cluster snapshot etcd save Signed-off-by: JoshVanL --- .../cmd/cluster_snapshot_consul_restore.go | 4 +- .../cmd/cluster_snapshot_consul_save.go | 4 +- .../cmd/cluster_snapshot_etcd_restore.go | 32 +++ cmd/tarmak/cmd/cluster_snapshot_etcd_save.go | 19 +- pkg/tarmak/snapshot/consul/consul.go | 3 +- pkg/tarmak/snapshot/etcd/etcd.go | 205 ++++++++++++++++++ pkg/tarmak/snapshot/snapshot.go | 3 +- puppet/modules/etcd/manifests/install.pp | 5 +- 8 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go create mode 100644 pkg/tarmak/snapshot/etcd/etcd.go diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go index 6f0939f33c..756d73c951 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go @@ -22,8 +22,8 @@ var clusterSnapshotConsulRestoreCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - c := consul.NewConsul(t, args[0]) - t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, c).Restore) + s := consul.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Restore) }, } diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_save.go b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go index 9718abab5c..84c99aea75 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_consul_save.go +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go @@ -22,8 +22,8 @@ var clusterSnapshotConsulSaveCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - c := consul.NewConsul(t, args[0]) - t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, c).Save) + s := consul.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Save) }, } diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go new file mode 100644 index 0000000000..b44869fd03 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go @@ -0,0 +1,32 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/etcd" +) + +var clusterSnapshotEtcdRestoreCmd = &cobra.Command{ + Use: "restore [source path]", + Short: "restore etcd cluster with source snapshot", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single source path, got=%d", len(args)) + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + s := etcd.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Restore) + }, +} + +func init() { + clusterSnapshotEtcdCmd.AddCommand(clusterSnapshotEtcdRestoreCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go index 3438142e35..4c3e09e392 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go @@ -2,13 +2,28 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/etcd" ) var clusterSnapshotEtcdSaveCmd = &cobra.Command{ - Use: "save [target path]", - Short: "save etcd snapshot to target path", + Use: "save [target path prefix]", + Short: "save etcd snapshot to target path prefix, i.e 'backup-'", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single target path, got=%d", len(args)) + } + + return nil + }, Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + s := etcd.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Save) }, } diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go index 343a28fbc1..e07cbc89ea 100644 --- a/pkg/tarmak/snapshot/consul/consul.go +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -1,3 +1,4 @@ +// Copyright Jetstack Ltd. See LICENSE for details. package consul import ( @@ -36,7 +37,7 @@ type Consul struct { aliases []string } -func NewConsul(tarmak interfaces.Tarmak, path string) *Consul { +func New(tarmak interfaces.Tarmak, path string) *Consul { return &Consul{ tarmak: tarmak, ssh: tarmak.SSH(), diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go new file mode 100644 index 0000000000..9165a1063d --- /dev/null +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -0,0 +1,205 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package etcd + +import ( + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + //"github.com/docker/docker/pkg/archive" + "github.com/hashicorp/go-multierror" + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot" +) + +var _ interfaces.Snapshot = &Etcd{} + +const ( + snapshotTimeLayout = "2006-01-02_15-04-05" + + etcdctlCmd = "/opt/bin/etcdctl snapshot %s %s > /dev/null;" + tarCmd = "tar -czPf - %s" +) + +var ( + stores = []map[string]string{ + {"store": "k8s-main", "file": "k8s", "port": "2379"}, + {"store": "k8s-events", "file": "k8s", "port": "2369"}, + {"store": "overlay", "file": "overlay", "port": "2359"}, + } + + exportCmd = []string{ + "ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem", + "ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem", + "ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem", + "ETCDCTL_API=3", + "ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}}", + } +) + +type Etcd struct { + tarmak interfaces.Tarmak + ssh interfaces.SSH + log *logrus.Entry + ctx interfaces.CancellationContext + + path string + aliases []string + errLock sync.Mutex // prevent multiple writes to result error +} + +func New(tarmak interfaces.Tarmak, path string) *Etcd { + return &Etcd{ + tarmak: tarmak, + ssh: tarmak.SSH(), + ctx: tarmak.CancellationContext(), + log: tarmak.Log(), + path: path, + } +} + +func (e *Etcd) Save() error { + aliases, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeEtcd) + if err != nil { + return err + } + e.aliases = aliases + + e.log.Infof("saving snapshots from instance %s", aliases[0]) + + var wg sync.WaitGroup + var result *multierror.Error + + saveFunc := func(store map[string]string) { + defer wg.Done() + + hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", + store["store"], time.Now().Format(snapshotTimeLayout)) + targetPath := fmt.Sprintf("%s%s.db", e.path, store["store"]) + + cmdArgs := append(e.template(exportCmd, store), + strings.Split(fmt.Sprintf(etcdctlCmd, "save", hostPath), " ")...) + cmdArgs = append(cmdArgs, + strings.Split(fmt.Sprintf(tarCmd, hostPath), " ")...) + + reader, writer := io.Pipe() + go e.readTarFromStream(targetPath, reader, result) + + err = e.sshCmd( + aliases[0], + cmdArgs, + writer, + ) + if err != nil { + + e.errLock.Lock() + result = multierror.Append(result, err) + e.errLock.Unlock() + + return + } + + e.log.Infof("etcd %s snapshot saved to %s", store["store"], targetPath) + + select { + case <-e.ctx.Done(): + return + default: + } + } + + wg.Add(len(stores)) + + for _, store := range stores { + go saveFunc(store) + } + + wg.Wait() + + select { + case <-e.ctx.Done(): + return e.ctx.Err() + default: + } + + return result.ErrorOrNil() +} + +func (e *Etcd) Restore() error { + aliases, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeEtcd) + if err != nil { + return err + } + e.aliases = aliases + + return nil +} + +func (e *Etcd) sshCmd(host string, args []string, stdout io.Writer) error { + readerE, writerE := io.Pipe() + scannerE := bufio.NewScanner(readerE) + + go func() { + for scannerE.Scan() { + e.log.WithField("std", "err").Warn(scannerE.Text()) + } + }() + + args = append([]string{"sudo"}, args...) + ret, err := e.ssh.ExecuteWithWriter(host, args[0], args[1:], stdout, writerE) + if ret != 0 { + cmdStr := fmt.Sprintf("%s", strings.Join(args, " ")) + return fmt.Errorf("command [%s] returned non-zero: %d", cmdStr, ret) + } + + return err +} + +func (e *Etcd) readTarFromStream(dest string, stream io.Reader, result *multierror.Error) { + gzr, err := gzip.NewReader(stream) + if err != nil { + + e.errLock.Lock() + result = multierror.Append(result, err) + e.errLock.Unlock() + + return + } + + f, err := os.Create(dest) + if err != nil { + + e.errLock.Lock() + result = multierror.Append(result, err) + e.errLock.Unlock() + + return + } + + if _, err := io.Copy(f, gzr); err != nil { + + e.errLock.Lock() + result = multierror.Append(result, err) + e.errLock.Unlock() + + return + } +} + +func (e *Etcd) template(args []string, vars map[string]string) []string { + for i := range args { + for k, v := range vars { + args[i] = strings.Replace(args[i], fmt.Sprintf("{{%s}}", k), v, -1) + } + } + + return args +} diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index c78e3fd809..5317bdd7b4 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -1,3 +1,4 @@ +// Copyright Jetstack Ltd. See LICENSE for details. package snapshot import ( @@ -41,7 +42,7 @@ func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error } if len(aliases) == 0 { - return nil, fmt.Errorf("no host aliases were found with the role %s", role) + return nil, fmt.Errorf("no host aliases were found with role %s", role) } return aliases, result.ErrorOrNil() diff --git a/puppet/modules/etcd/manifests/install.pp b/puppet/modules/etcd/manifests/install.pp index ccd25240e7..062cbdf6b6 100644 --- a/puppet/modules/etcd/manifests/install.pp +++ b/puppet/modules/etcd/manifests/install.pp @@ -33,5 +33,8 @@ creates => "${dest_dir}/etcd", path => ['/usr/bin/', '/bin'], } - + -> file { "${bin_dir}/etcdctl": + ensure => 'link', + target => "${dest_dir}/etcdctl", + } } From f2186613119418783a7a9c32f3ec4763b543388d Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Wed, 7 Nov 2018 19:40:52 +0000 Subject: [PATCH 11/15] Attempt to tar both received and sent snapshots Signed-off-by: JoshVanL --- pkg/tarmak/snapshot/consul/consul.go | 80 +++++++++++++++------------- pkg/tarmak/snapshot/etcd/etcd.go | 64 +++++----------------- pkg/tarmak/snapshot/snapshot.go | 66 +++++++++++++++++++++++ 3 files changed, 122 insertions(+), 88 deletions(-) diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go index e07cbc89ea..fd3812b7ab 100644 --- a/pkg/tarmak/snapshot/consul/consul.go +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -5,9 +5,12 @@ import ( "bufio" "fmt" "io" + "os" "strings" + "sync" "time" + "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" @@ -18,14 +21,11 @@ import ( var _ interfaces.Snapshot = &Consul{} const ( - snapshotTimeLayout = "2006-01-02_15-04-05" + consulCmd = "consul snapshot %s %s > /dev/null;" ) var ( - exportCmd = []string{ - "export", - "CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token');", - } + envCmd = []string{"CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token')"} ) type Consul struct { @@ -54,27 +54,33 @@ func (c *Consul) Save() error { c.aliases = aliases c.log.Infof("saving snapshot from instance %s", aliases[0]) - targetPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshotTimeLayout)) - cmdArgs := append(exportCmd, "consul", "snapshot", "save", targetPath) + var result *multierror.Error + var errLock sync.Mutex + + reader, writer := io.Pipe() + go snapshot.ReadTarFromStream(c.path, reader, result, errLock) + + hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", + time.Now().Format(snapshot.TimeLayout)) + cmdArgs := append(envCmd, + strings.Split(fmt.Sprintf(consulCmd, "save", hostPath), " ")...) + cmdArgs = append(cmdArgs, + strings.Split(fmt.Sprintf(snapshot.TarCCmd, hostPath), " ")...) + err = c.sshCmd( aliases[0], - cmdArgs[0], - cmdArgs[1:], + cmdArgs, + nil, + writer, ) if err != nil { return err } - ret, err := c.ssh.ScpToLocal(aliases[0], targetPath, c.path) - if ret != 0 { - cmdStr := fmt.Sprintf("%s", strings.Join(cmdArgs, " ")) - return fmt.Errorf("command [%s] returned non-zero: %d, %s", cmdStr, ret, err) - } - c.log.Infof("consul snapshot saved to %s", c.path) - return err + return nil } func (c *Consul) Restore() error { @@ -86,18 +92,27 @@ func (c *Consul) Restore() error { for _, a := range aliases { c.log.Infof("restoring snapshot to instance %s", a) - targetPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshotTimeLayout)) - ret, err := c.ssh.ScpToHost(a, c.path, targetPath) - if ret != 0 { - return fmt.Errorf("command scp returned non-zero: %d, %s", ret, err) - } + hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", + time.Now().Format(snapshot.TimeLayout)) + + cmdArgs := strings.Split(fmt.Sprintf(snapshot.TarXCmd, hostPath), " ") + //cmdArgs := strings.Split(snapshot.TarXCmd, " ") + //cmdArgs = append(cmdArgs, + // append(envCmd, + // strings.Split(fmt.Sprintf(consulCmd, "restore", hostPath), " ")...)...) + + var result *multierror.Error + var errLock sync.Mutex + + reader, writer := io.Pipe() + go snapshot.WriteTarToStream(c.path, writer, result, errLock) - cmdArgs := append(exportCmd, "consul", "snapshot", "restore", targetPath) err = c.sshCmd( a, - cmdArgs[0], - cmdArgs[1:], + cmdArgs, + reader, + os.Stdout, ) if err != nil { return err @@ -106,31 +121,22 @@ func (c *Consul) Restore() error { c.log.Infof("consul snapshot restored from %s", c.path) - return err + return nil } -func (c *Consul) sshCmd(host, command string, args []string) error { - readerO, writerO := io.Pipe() +func (c *Consul) sshCmd(host string, args []string, stdin io.Reader, stdout io.Writer) error { readerE, writerE := io.Pipe() - scannerO := bufio.NewScanner(readerO) scannerE := bufio.NewScanner(readerE) - go func() { - for scannerO.Scan() { - c.log.WithField("std", "out").Debug(scannerO.Text()) - } - }() - go func() { for scannerE.Scan() { c.log.WithField("std", "err").Warn(scannerE.Text()) } }() - ret, err := c.ssh.ExecuteWithWriter(host, command, args, writerO, writerE) + ret, err := c.ssh.ExecuteWithPipe(host, args[0], args[1:], stdin, stdout, writerE) if ret != 0 { - cmdStr := fmt.Sprintf("%s %s", command, strings.Join(args, " ")) - return fmt.Errorf("command [%s] returned non-zero: %d", cmdStr, ret) + return fmt.Errorf("command [%s] returned non-zero: %d", strings.Join(args, " "), ret) } return err diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go index 9165a1063d..efe3640320 100644 --- a/pkg/tarmak/snapshot/etcd/etcd.go +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -3,15 +3,12 @@ package etcd import ( "bufio" - "compress/gzip" "fmt" "io" - "os" "strings" "sync" "time" - //"github.com/docker/docker/pkg/archive" "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" @@ -23,10 +20,7 @@ import ( var _ interfaces.Snapshot = &Etcd{} const ( - snapshotTimeLayout = "2006-01-02_15-04-05" - etcdctlCmd = "/opt/bin/etcdctl snapshot %s %s > /dev/null;" - tarCmd = "tar -czPf - %s" ) var ( @@ -36,7 +30,7 @@ var ( {"store": "overlay", "file": "overlay", "port": "2359"}, } - exportCmd = []string{ + envCmd = []string{ "ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem", "ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem", "ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem", @@ -53,7 +47,6 @@ type Etcd struct { path string aliases []string - errLock sync.Mutex // prevent multiple writes to result error } func New(tarmak interfaces.Tarmak, path string) *Etcd { @@ -77,21 +70,22 @@ func (e *Etcd) Save() error { var wg sync.WaitGroup var result *multierror.Error + var errLock sync.Mutex saveFunc := func(store map[string]string) { defer wg.Done() - hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", - store["store"], time.Now().Format(snapshotTimeLayout)) targetPath := fmt.Sprintf("%s%s.db", e.path, store["store"]) - cmdArgs := append(e.template(exportCmd, store), + reader, writer := io.Pipe() + go snapshot.ReadTarFromStream(targetPath, reader, result, errLock) + + hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", + store["store"], time.Now().Format(snapshot.TimeLayout)) + cmdArgs := append(e.template(envCmd, store), strings.Split(fmt.Sprintf(etcdctlCmd, "save", hostPath), " ")...) cmdArgs = append(cmdArgs, - strings.Split(fmt.Sprintf(tarCmd, hostPath), " ")...) - - reader, writer := io.Pipe() - go e.readTarFromStream(targetPath, reader, result) + strings.Split(fmt.Sprintf(snapshot.TarCCmd, hostPath), " ")...) err = e.sshCmd( aliases[0], @@ -100,9 +94,9 @@ func (e *Etcd) Save() error { ) if err != nil { - e.errLock.Lock() + errLock.Lock() result = multierror.Append(result, err) - e.errLock.Unlock() + errLock.Unlock() return } @@ -154,46 +148,14 @@ func (e *Etcd) sshCmd(host string, args []string, stdout io.Writer) error { }() args = append([]string{"sudo"}, args...) - ret, err := e.ssh.ExecuteWithWriter(host, args[0], args[1:], stdout, writerE) + ret, err := e.ssh.ExecuteWithPipe(host, args[0], args[1:], nil, stdout, writerE) if ret != 0 { - cmdStr := fmt.Sprintf("%s", strings.Join(args, " ")) - return fmt.Errorf("command [%s] returned non-zero: %d", cmdStr, ret) + return fmt.Errorf("command [%s] returned non-zero: %d", strings.Join(args, " "), ret) } return err } -func (e *Etcd) readTarFromStream(dest string, stream io.Reader, result *multierror.Error) { - gzr, err := gzip.NewReader(stream) - if err != nil { - - e.errLock.Lock() - result = multierror.Append(result, err) - e.errLock.Unlock() - - return - } - - f, err := os.Create(dest) - if err != nil { - - e.errLock.Lock() - result = multierror.Append(result, err) - e.errLock.Unlock() - - return - } - - if _, err := io.Copy(f, gzr); err != nil { - - e.errLock.Lock() - result = multierror.Append(result, err) - e.errLock.Unlock() - - return - } -} - func (e *Etcd) template(args []string, vars map[string]string) []string { for i := range args { for k, v := range vars { diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index 5317bdd7b4..6b350f25ae 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -2,7 +2,11 @@ package snapshot import ( + "compress/gzip" "fmt" + "io" + "os" + "sync" "github.com/hashicorp/go-multierror" @@ -10,6 +14,13 @@ import ( "github.com/jetstack/tarmak/pkg/tarmak/utils" ) +const ( + TimeLayout = "2006-01-02_15-04-05" + TarCCmd = "tar -czPf - %s" + //TarXCmd = "cat | tar -xz | cat > /tmp/foo" //| cat > %s" + TarXCmd = "cat > %s" +) + func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error) { if err := tarmak.SSH().WriteConfig(tarmak.Cluster()); err != nil { return nil, err @@ -47,3 +58,58 @@ func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error return aliases, result.ErrorOrNil() } + +func ReadTarFromStream(dest string, stream io.Reader, result *multierror.Error, errLock sync.Mutex) { + gzr, err := gzip.NewReader(stream) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + f, err := os.Create(dest) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + if _, err := io.Copy(f, gzr); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } +} + +func WriteTarToStream(src string, stream io.WriteCloser, result *multierror.Error, errLock sync.Mutex) { + //defer stream.Close() + + f, err := os.Open(src) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + gzw := gzip.NewWriter(stream) + if _, err := io.Copy(gzw, f); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } +} From ec962f27778b25cd783c4d36e2100f85a92d4a13 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Fri, 9 Nov 2018 17:23:41 +0000 Subject: [PATCH 12/15] Gzip files Signed-off-by: JoshVanL --- pkg/tarmak/snapshot/consul/consul.go | 11 +++++------ pkg/tarmak/snapshot/etcd/etcd.go | 2 +- pkg/tarmak/snapshot/snapshot.go | 12 +++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go index fd3812b7ab..d6019d23e9 100644 --- a/pkg/tarmak/snapshot/consul/consul.go +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -66,7 +66,7 @@ func (c *Consul) Save() error { cmdArgs := append(envCmd, strings.Split(fmt.Sprintf(consulCmd, "save", hostPath), " ")...) cmdArgs = append(cmdArgs, - strings.Split(fmt.Sprintf(snapshot.TarCCmd, hostPath), " ")...) + strings.Split(fmt.Sprintf(snapshot.GZipCCmd, hostPath), " ")...) err = c.sshCmd( aliases[0], @@ -96,11 +96,10 @@ func (c *Consul) Restore() error { hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshot.TimeLayout)) - cmdArgs := strings.Split(fmt.Sprintf(snapshot.TarXCmd, hostPath), " ") - //cmdArgs := strings.Split(snapshot.TarXCmd, " ") - //cmdArgs = append(cmdArgs, - // append(envCmd, - // strings.Split(fmt.Sprintf(consulCmd, "restore", hostPath), " ")...)...) + cmdArgs := strings.Split(fmt.Sprintf(snapshot.GZipDCmd, hostPath, hostPath), " ") + cmdArgs = append(cmdArgs, + append(envCmd, + strings.Split(fmt.Sprintf(consulCmd, "restore", hostPath), " ")...)...) var result *multierror.Error var errLock sync.Mutex diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go index efe3640320..015024d1cc 100644 --- a/pkg/tarmak/snapshot/etcd/etcd.go +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -85,7 +85,7 @@ func (e *Etcd) Save() error { cmdArgs := append(e.template(envCmd, store), strings.Split(fmt.Sprintf(etcdctlCmd, "save", hostPath), " ")...) cmdArgs = append(cmdArgs, - strings.Split(fmt.Sprintf(snapshot.TarCCmd, hostPath), " ")...) + strings.Split(fmt.Sprintf(snapshot.GZipCCmd, hostPath), " ")...) err = e.sshCmd( aliases[0], diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index 6b350f25ae..cc9362c0c9 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -16,9 +16,8 @@ import ( const ( TimeLayout = "2006-01-02_15-04-05" - TarCCmd = "tar -czPf - %s" - //TarXCmd = "cat | tar -xz | cat > /tmp/foo" //| cat > %s" - TarXCmd = "cat > %s" + GZipCCmd = "gzip -c %s;" + GZipDCmd = "cat > %s.gz; gzip -d %s.gz;" ) func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error) { @@ -79,6 +78,7 @@ func ReadTarFromStream(dest string, stream io.Reader, result *multierror.Error, return } + defer f.Close() if _, err := io.Copy(f, gzr); err != nil { @@ -91,8 +91,7 @@ func ReadTarFromStream(dest string, stream io.Reader, result *multierror.Error, } func WriteTarToStream(src string, stream io.WriteCloser, result *multierror.Error, errLock sync.Mutex) { - //defer stream.Close() - + defer stream.Close() f, err := os.Open(src) if err != nil { @@ -102,8 +101,11 @@ func WriteTarToStream(src string, stream io.WriteCloser, result *multierror.Erro return } + defer f.Close() gzw := gzip.NewWriter(stream) + defer gzw.Close() + if _, err := io.Copy(gzw, f); err != nil { errLock.Lock() From 9f83889b7689cf4c4adf72abc7dd85f78c3f1847 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 27 Nov 2018 16:37:27 +0000 Subject: [PATCH 13/15] Use ssh package to save backups Signed-off-by: JoshVanL --- .../cmd/cluster_snapshot_etcd_restore.go | 2 +- cmd/tarmak/cmd/cluster_snapshot_etcd_save.go | 12 +-- cmd/tarmak/cmd/cluster_ssh.go | 3 +- pkg/tarmak/environment/bastion.go | 2 +- pkg/tarmak/interfaces/interfaces.go | 6 +- pkg/tarmak/snapshot/consul/consul.go | 62 ++++++-------- pkg/tarmak/snapshot/etcd/etcd.go | 85 +++++++++---------- pkg/tarmak/snapshot/snapshot.go | 75 +++++++++++++--- pkg/tarmak/ssh.go | 2 +- pkg/tarmak/ssh/ssh.go | 9 +- pkg/tarmak/ssh/tunnel.go | 14 ++- puppet/modules/etcd/manifests/init.pp | 1 + puppet/modules/etcd/manifests/install.pp | 2 +- 13 files changed, 156 insertions(+), 119 deletions(-) diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go index b44869fd03..74cb1940eb 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go @@ -15,7 +15,7 @@ var clusterSnapshotEtcdRestoreCmd = &cobra.Command{ Short: "restore etcd cluster with source snapshot", PreRunE: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { - return fmt.Errorf("expecting single source path, got=%d", len(args)) + return fmt.Errorf("expecting single target path, got=%d", len(args)) } return nil diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go index 4c3e09e392..62f1ffdc79 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go @@ -2,8 +2,6 @@ package cmd import ( - "fmt" - "github.com/spf13/cobra" "github.com/jetstack/tarmak/pkg/tarmak" @@ -13,14 +11,10 @@ import ( var clusterSnapshotEtcdSaveCmd = &cobra.Command{ Use: "save [target path prefix]", Short: "save etcd snapshot to target path prefix, i.e 'backup-'", - PreRunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("expecting single target path, got=%d", len(args)) - } - - return nil - }, Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + args = []string{""} + } t := tarmak.New(globalFlags) s := etcd.New(t, args[0]) t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Save) diff --git a/cmd/tarmak/cmd/cluster_ssh.go b/cmd/tarmak/cmd/cluster_ssh.go index d16c66bd8c..a2a7087389 100644 --- a/cmd/tarmak/cmd/cluster_ssh.go +++ b/cmd/tarmak/cmd/cluster_ssh.go @@ -3,6 +3,7 @@ package cmd import ( "errors" + "strings" "github.com/spf13/cobra" @@ -20,7 +21,7 @@ var clusterSshCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - t.Perform(t.SSHPassThrough(args[0], args[1:])) + t.Perform(t.SSHPassThrough(args[0], strings.Join(args[1:], " "))) }, } diff --git a/pkg/tarmak/environment/bastion.go b/pkg/tarmak/environment/bastion.go index 602dc5f906..6700de3b19 100644 --- a/pkg/tarmak/environment/bastion.go +++ b/pkg/tarmak/environment/bastion.go @@ -61,7 +61,7 @@ func (e *Environment) VerifyBastionAvailable() error { executeSSH := func() error { retCode, err := ssh.Execute( "bastion", - []string{"/bin/true"}, + "/bin/true", nil, nil, stderrW, ) diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index db94d6c6b1..0b4e5e1a97 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -208,9 +208,9 @@ type Terraform interface { type SSH interface { WriteConfig(Cluster) error - PassThrough(host string, additionalArguments []string) error + PassThrough(host string, additionalArguments string) error Tunnel(destination, destinationPort, localPort string, daemonize bool) Tunnel - Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (returnCode int, err error) + Execute(host string, cmd string, stdin io.Reader, stdout, stderr io.Writer) (returnCode int, err error) Validate() error Cleanup() } @@ -296,4 +296,6 @@ type CancellationContext interface { type Snapshot interface { Save() error Restore() error + Log() *logrus.Entry + SSH() SSH } diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go index d6019d23e9..99407ff559 100644 --- a/pkg/tarmak/snapshot/consul/consul.go +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -2,7 +2,6 @@ package consul import ( - "bufio" "fmt" "io" "os" @@ -55,25 +54,26 @@ func (c *Consul) Save() error { c.log.Infof("saving snapshot from instance %s", aliases[0]) - var result *multierror.Error - var errLock sync.Mutex - - reader, writer := io.Pipe() - go snapshot.ReadTarFromStream(c.path, reader, result, errLock) - hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshot.TimeLayout)) - cmdArgs := append(envCmd, - strings.Split(fmt.Sprintf(consulCmd, "save", hostPath), " ")...) - cmdArgs = append(cmdArgs, - strings.Split(fmt.Sprintf(snapshot.GZipCCmd, hostPath), " ")...) - - err = c.sshCmd( - aliases[0], - cmdArgs, - nil, - writer, - ) + cmdArgs := fmt.Sprintf(` +export CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token'); +export DATACENTER=$(sudo cat /etc/consul/consul.json | jq -r '.datacenter'); +/usr/local/bin/consul snapshot save -datacenter $DATACENTER %s; +/usr/local/bin/consul snapshot inspect %s`, hostPath, hostPath) + + err = snapshot.SSHCmd(c, aliases[0], cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + reader, writer := io.Pipe() + err = snapshot.TarFromStream(func() error { + err := snapshot.SSHCmd(c, aliases[0], fmt.Sprintf(snapshot.GZipCCmd, hostPath), + nil, writer, nil) + writer.Close() + return err + }, reader, c.path) if err != nil { return err } @@ -107,11 +107,11 @@ func (c *Consul) Restore() error { reader, writer := io.Pipe() go snapshot.WriteTarToStream(c.path, writer, result, errLock) - err = c.sshCmd( - a, - cmdArgs, + err = snapshot.SSHCmd(c, a, + strings.Join(cmdArgs, " "), reader, os.Stdout, + nil, ) if err != nil { return err @@ -123,20 +123,10 @@ func (c *Consul) Restore() error { return nil } -func (c *Consul) sshCmd(host string, args []string, stdin io.Reader, stdout io.Writer) error { - readerE, writerE := io.Pipe() - scannerE := bufio.NewScanner(readerE) - - go func() { - for scannerE.Scan() { - c.log.WithField("std", "err").Warn(scannerE.Text()) - } - }() - - ret, err := c.ssh.ExecuteWithPipe(host, args[0], args[1:], stdin, stdout, writerE) - if ret != 0 { - return fmt.Errorf("command [%s] returned non-zero: %d", strings.Join(args, " "), ret) - } +func (c *Consul) Log() *logrus.Entry { + return c.log +} - return err +func (c *Consul) SSH() interfaces.SSH { + return c.ssh } diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go index 015024d1cc..a64c27794b 100644 --- a/pkg/tarmak/snapshot/etcd/etcd.go +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -2,7 +2,7 @@ package etcd import ( - "bufio" + //"bufio" "fmt" "io" "strings" @@ -20,7 +20,7 @@ import ( var _ interfaces.Snapshot = &Etcd{} const ( - etcdctlCmd = "/opt/bin/etcdctl snapshot %s %s > /dev/null;" + etcdctlCmd = `/opt/bin/etcdctl snapshot %s %s` ) var ( @@ -30,13 +30,13 @@ var ( {"store": "overlay", "file": "overlay", "port": "2359"}, } - envCmd = []string{ - "ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem", - "ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem", - "ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem", - "ETCDCTL_API=3", - "ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}}", - } + envCmd = ` +export ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem; +export ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem; +export ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem; +export ETCDCTL_API=3; +export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}} +` ) type Etcd struct { @@ -75,23 +75,12 @@ func (e *Etcd) Save() error { saveFunc := func(store map[string]string) { defer wg.Done() - targetPath := fmt.Sprintf("%s%s.db", e.path, store["store"]) - - reader, writer := io.Pipe() - go snapshot.ReadTarFromStream(targetPath, reader, result, errLock) - hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", store["store"], time.Now().Format(snapshot.TimeLayout)) - cmdArgs := append(e.template(envCmd, store), - strings.Split(fmt.Sprintf(etcdctlCmd, "save", hostPath), " ")...) - cmdArgs = append(cmdArgs, - strings.Split(fmt.Sprintf(snapshot.GZipCCmd, hostPath), " ")...) - - err = e.sshCmd( - aliases[0], - cmdArgs, - writer, - ) + + cmdArgs := fmt.Sprintf(`sudo /bin/bash -c "%s; %s"`, e.template(envCmd, store), + fmt.Sprintf(etcdctlCmd, "save", hostPath)) + err = snapshot.SSHCmd(e, aliases[0], cmdArgs, nil, nil, nil) if err != nil { errLock.Lock() @@ -101,6 +90,22 @@ func (e *Etcd) Save() error { return } + targetPath := fmt.Sprintf("%s%s.db", e.path, store["store"]) + reader, writer := io.Pipe() + err = snapshot.TarFromStream(func() error { + err := snapshot.SSHCmd(e, aliases[0], fmt.Sprintf(snapshot.GZipCCmd, hostPath), + nil, writer, nil) + writer.Close() + return err + }, reader, targetPath) + if err != nil { + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + e.log.Infof("etcd %s snapshot saved to %s", store["store"], targetPath) select { @@ -111,7 +116,6 @@ func (e *Etcd) Save() error { } wg.Add(len(stores)) - for _, store := range stores { go saveFunc(store) } @@ -137,31 +141,18 @@ func (e *Etcd) Restore() error { return nil } -func (e *Etcd) sshCmd(host string, args []string, stdout io.Writer) error { - readerE, writerE := io.Pipe() - scannerE := bufio.NewScanner(readerE) - - go func() { - for scannerE.Scan() { - e.log.WithField("std", "err").Warn(scannerE.Text()) - } - }() - - args = append([]string{"sudo"}, args...) - ret, err := e.ssh.ExecuteWithPipe(host, args[0], args[1:], nil, stdout, writerE) - if ret != 0 { - return fmt.Errorf("command [%s] returned non-zero: %d", strings.Join(args, " "), ret) +func (e *Etcd) template(args string, vars map[string]string) string { + for k, v := range vars { + args = strings.Replace(args, fmt.Sprintf("{{%s}}", k), v, -1) } - return err + return args } -func (e *Etcd) template(args []string, vars map[string]string) []string { - for i := range args { - for k, v := range vars { - args[i] = strings.Replace(args[i], fmt.Sprintf("{{%s}}", k), v, -1) - } - } +func (e *Etcd) Log() *logrus.Entry { + return e.log +} - return args +func (e *Etcd) SSH() interfaces.SSH { + return e.ssh } diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index cc9362c0c9..458dff4bbf 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -2,6 +2,7 @@ package snapshot import ( + "bufio" "compress/gzip" "fmt" "io" @@ -16,7 +17,7 @@ import ( const ( TimeLayout = "2006-01-02_15-04-05" - GZipCCmd = "gzip -c %s;" + GZipCCmd = "gzip -c %s" GZipDCmd = "cat > %s.gz; gzip -d %s.gz;" ) @@ -58,27 +59,40 @@ func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error return aliases, result.ErrorOrNil() } -func ReadTarFromStream(dest string, stream io.Reader, result *multierror.Error, errLock sync.Mutex) { - gzr, err := gzip.NewReader(stream) +func TarFromStream(sshCmd func() error, stream io.ReadCloser, path string) error { + var result *multierror.Error + var errLock sync.Mutex + var wg sync.WaitGroup + + f, err := os.Create(path) if err != nil { + return err + } + defer f.Close() - errLock.Lock() - result = multierror.Append(result, err) - errLock.Unlock() + wg.Add(1) + go func() { + defer wg.Done() + err = sshCmd() + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + } return - } + }() - f, err := os.Create(dest) + gzr, err := gzip.NewReader(stream) if err != nil { errLock.Lock() result = multierror.Append(result, err) errLock.Unlock() - return } - defer f.Close() + defer gzr.Close() if _, err := io.Copy(f, gzr); err != nil { @@ -86,11 +100,18 @@ func ReadTarFromStream(dest string, stream io.Reader, result *multierror.Error, result = multierror.Append(result, err) errLock.Unlock() - return } + + wg.Wait() + + if result != nil { + return result + } + + return nil } -func WriteTarToStream(src string, stream io.WriteCloser, result *multierror.Error, errLock sync.Mutex) { +func WriteTarToStream(src string, stream io.WriteCloser, result error, errLock sync.Mutex) { defer stream.Close() f, err := os.Open(src) if err != nil { @@ -115,3 +136,33 @@ func WriteTarToStream(src string, stream io.WriteCloser, result *multierror.Erro return } } + +func SSHCmd(s interfaces.Snapshot, host, cmd string, stdin io.Reader, stdout, stderr io.Writer) error { + for _, w := range []struct { + writer io.Writer + out string + }{ + {stdout, "out"}, + {stderr, "err"}, + } { + + if w.writer == nil { + var reader *io.PipeReader + reader, w.writer = io.Pipe() + scanner := bufio.NewScanner(reader) + + go func() { + for scanner.Scan() { + s.Log().WithField("std", w.out).Warn(scanner.Text()) + } + }() + } + } + + ret, err := s.SSH().Execute(host, cmd, stdin, stdout, stderr) + if ret != 0 { + return fmt.Errorf("command [%s] returned non-zero (%d): %s", cmd, ret, err) + } + + return err +} diff --git a/pkg/tarmak/ssh.go b/pkg/tarmak/ssh.go index c9b1fedcf1..789dcd2e5b 100644 --- a/pkg/tarmak/ssh.go +++ b/pkg/tarmak/ssh.go @@ -9,7 +9,7 @@ func (t *Tarmak) SSH() interfaces.SSH { return t.ssh } -func (t *Tarmak) SSHPassThrough(host string, argsAdditional []string) error { +func (t *Tarmak) SSHPassThrough(host string, argsAdditional string) error { if err := t.ssh.WriteConfig(t.Cluster()); err != nil { return err } diff --git a/pkg/tarmak/ssh/ssh.go b/pkg/tarmak/ssh/ssh.go index 1316cb98d6..0c9d4af651 100644 --- a/pkg/tarmak/ssh/ssh.go +++ b/pkg/tarmak/ssh/ssh.go @@ -11,7 +11,6 @@ import ( "net" "os" "path/filepath" - "strings" "time" "github.com/sirupsen/logrus" @@ -79,8 +78,8 @@ func (s *SSH) WriteConfig(c interfaces.Cluster) error { } // Pass through a local CLI session -func (s *SSH) PassThrough(hostName string, argsAdditional []string) error { - if len(argsAdditional) > 0 { +func (s *SSH) PassThrough(hostName string, argsAdditional string) error { + if argsAdditional != "" { _, err := s.Execute(hostName, argsAdditional, nil, nil, nil) return err } @@ -137,7 +136,7 @@ func (s *SSH) PassThrough(hostName string, argsAdditional []string) error { return nil } -func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { +func (s *SSH) Execute(host string, cmd string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { client, err := s.client(host) if err != nil { return -1, err @@ -168,7 +167,7 @@ func (s *SSH) Execute(host string, cmd []string, stdin io.Reader, stdout, stderr sess.Stdin = stdin } - err = sess.Start(strings.Join(cmd, " ")) + err = sess.Start(cmd) if err != nil { return -1, err } diff --git a/pkg/tarmak/ssh/tunnel.go b/pkg/tarmak/ssh/tunnel.go index 8084e334f1..8ec1669633 100644 --- a/pkg/tarmak/ssh/tunnel.go +++ b/pkg/tarmak/ssh/tunnel.go @@ -55,7 +55,7 @@ func (s *SSH) Tunnel(dest, destPort, localPort string, daemonize bool) interface // Start tunnel and wait till a tcp socket is reachable func (t *Tunnel) Start() error { // ensure there is connectivity to the bastion - ret, err := t.ssh.Execute("bastion", []string{"/bin/true"}, nil, nil, nil) + ret, err := t.ssh.Execute("bastion", "/bin/true", nil, nil, nil) if err != nil || ret != 0 { return fmt.Errorf("error checking SSH connecting to bastion (%d): %s", ret, err) } @@ -99,12 +99,20 @@ func (t *Tunnel) Start() error { } func (t *Tunnel) handle() { + tries := 5 for { remoteConn, err := t.serverConn.Dial("tcp", net.JoinHostPort(t.dest, t.destPort)) if err != nil { - t.log.Errorf("failed to create tunnel: %s", err) - return + t.log.Warnf("failed to create tunnel to remote connection: %s", err) + tries-- + if tries == 0 { + t.log.Error("failed to create tunnel after 5 tries") + return + } + + time.Sleep(time.Second * 3) + continue } t.remoteConns = append(t.remoteConns, remoteConn) diff --git a/puppet/modules/etcd/manifests/init.pp b/puppet/modules/etcd/manifests/init.pp index 1dc08d0d7f..fd4b83f35a 100644 --- a/puppet/modules/etcd/manifests/init.pp +++ b/puppet/modules/etcd/manifests/init.pp @@ -13,6 +13,7 @@ $gid = $::etcd::params::gid, $user = $::etcd::params::user, $group = $::etcd::params::group, + $bin_dir = $::etcd::params::bin_dir, Boolean $backup_enabled = false, Enum['aws:kms',''] $backup_sse = '', String $backup_bucket_prefix = '', diff --git a/puppet/modules/etcd/manifests/install.pp b/puppet/modules/etcd/manifests/install.pp index 062cbdf6b6..871e70fca1 100644 --- a/puppet/modules/etcd/manifests/install.pp +++ b/puppet/modules/etcd/manifests/install.pp @@ -33,7 +33,7 @@ creates => "${dest_dir}/etcd", path => ['/usr/bin/', '/bin'], } - -> file { "${bin_dir}/etcdctl": + -> file { "${::etcd::bin_dir}/etcdctl": ensure => 'link', target => "${dest_dir}/etcdctl", } From 3994a9f1ce5c610158196bd99f42588a42bb0f6d Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 27 Nov 2018 17:48:02 +0000 Subject: [PATCH 14/15] Restore consul cluster by snapshot Signed-off-by: JoshVanL --- pkg/tarmak/snapshot/consul/consul.go | 37 +++++++++++------------- pkg/tarmak/snapshot/etcd/etcd.go | 5 ++-- pkg/tarmak/snapshot/snapshot.go | 43 +++++++++++++++++++++------- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go index 99407ff559..dffdf3b52d 100644 --- a/pkg/tarmak/snapshot/consul/consul.go +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -4,12 +4,8 @@ package consul import ( "fmt" "io" - "os" - "strings" - "sync" "time" - "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" @@ -56,7 +52,7 @@ func (c *Consul) Save() error { hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshot.TimeLayout)) - cmdArgs := fmt.Sprintf(` + cmdArgs := fmt.Sprintf(`set -e; export CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token'); export DATACENTER=$(sudo cat /etc/consul/consul.json | jq -r '.datacenter'); /usr/local/bin/consul snapshot save -datacenter $DATACENTER %s; @@ -93,26 +89,27 @@ func (c *Consul) Restore() error { for _, a := range aliases { c.log.Infof("restoring snapshot to instance %s", a) + reader, writer := io.Pipe() hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", time.Now().Format(snapshot.TimeLayout)) - cmdArgs := strings.Split(fmt.Sprintf(snapshot.GZipDCmd, hostPath, hostPath), " ") - cmdArgs = append(cmdArgs, - append(envCmd, - strings.Split(fmt.Sprintf(consulCmd, "restore", hostPath), " ")...)...) + err = snapshot.TarToStream(func() error { + err := snapshot.SSHCmd(c, a, fmt.Sprintf(snapshot.GZipDCmd, hostPath), reader, nil, nil) + return err + }, writer, c.path) + if err != nil { + return err + } - var result *multierror.Error - var errLock sync.Mutex + cmdArgs := fmt.Sprintf(`set -e; +export CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token'); +export DATACENTER=$(sudo cat /etc/consul/consul.json | jq -r '.datacenter'); +/usr/local/bin/consul snapshot inspect %s; +/usr/local/bin/consul snapshot restore -datacenter $DATACENTER %s; +echo number of keys: $(curl --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" -s 'http://127.0.0.1:8500/v1/kv/?keys' | jq '. | length'); +`, hostPath, hostPath) - reader, writer := io.Pipe() - go snapshot.WriteTarToStream(c.path, writer, result, errLock) - - err = snapshot.SSHCmd(c, a, - strings.Join(cmdArgs, " "), - reader, - os.Stdout, - nil, - ) + err = snapshot.SSHCmd(c, a, cmdArgs, nil, nil, nil) if err != nil { return err } diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go index a64c27794b..3bc86b87f3 100644 --- a/pkg/tarmak/snapshot/etcd/etcd.go +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -31,11 +31,12 @@ var ( } envCmd = ` +set -e; export ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem; export ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem; export ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem; export ETCDCTL_API=3; -export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}} +export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}}; ` ) @@ -78,7 +79,7 @@ func (e *Etcd) Save() error { hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", store["store"], time.Now().Format(snapshot.TimeLayout)) - cmdArgs := fmt.Sprintf(`sudo /bin/bash -c "%s; %s"`, e.template(envCmd, store), + cmdArgs := fmt.Sprintf(`sudo /bin/bash -c "%s %s"`, e.template(envCmd, store), fmt.Sprintf(etcdctlCmd, "save", hostPath)) err = snapshot.SSHCmd(e, aliases[0], cmdArgs, nil, nil, nil) if err != nil { diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index 458dff4bbf..7888a36fee 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -18,7 +18,7 @@ import ( const ( TimeLayout = "2006-01-02_15-04-05" GZipCCmd = "gzip -c %s" - GZipDCmd = "cat > %s.gz; gzip -d %s.gz;" + GZipDCmd = "gzip -d > %s" ) func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error) { @@ -59,7 +59,7 @@ func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error return aliases, result.ErrorOrNil() } -func TarFromStream(sshCmd func() error, stream io.ReadCloser, path string) error { +func TarFromStream(sshCmd func() error, stream io.Reader, path string) error { var result *multierror.Error var errLock sync.Mutex var wg sync.WaitGroup @@ -111,30 +111,51 @@ func TarFromStream(sshCmd func() error, stream io.ReadCloser, path string) error return nil } -func WriteTarToStream(src string, stream io.WriteCloser, result error, errLock sync.Mutex) { - defer stream.Close() +func TarToStream(sshCmd func() error, stream io.WriteCloser, src string) error { + var result *multierror.Error + var errLock sync.Mutex + var wg sync.WaitGroup + f, err := os.Open(src) if err != nil { + return err + } + defer f.Close() - errLock.Lock() - result = multierror.Append(result, err) - errLock.Unlock() + wg.Add(1) + go func() { + defer wg.Done() + err = sshCmd() + if err != nil { + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } return - } - defer f.Close() + }() gzw := gzip.NewWriter(stream) defer gzw.Close() - if _, err := io.Copy(gzw, f); err != nil { errLock.Lock() result = multierror.Append(result, err) errLock.Unlock() - return } + + f.Close() + gzw.Close() + stream.Close() + wg.Wait() + + if result != nil { + return result + } + + return nil } func SSHCmd(s interfaces.Snapshot, host, cmd string, stdin io.Reader, stdout, stderr io.Writer) error { From 51f235e93696e67018dd15dd14cbd2a79a7d0e57 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Wed, 28 Nov 2018 15:54:13 +0000 Subject: [PATCH 15/15] Adds snapshot etcd restore Signed-off-by: JoshVanL --- cmd/tarmak/cmd/cluster.go | 25 +++ cmd/tarmak/cmd/cluster_apply.go | 2 - .../cmd/cluster_snapshot_etcd_restore.go | 22 +- docs/cmd-docs.rst | 35 +++ docs/generated/cmd/tarmak/tarmak_clusters.rst | 1 + .../cmd/tarmak/tarmak_clusters_snapshot.rst | 40 ++++ .../tarmak_clusters_snapshot_consul.rst | 40 ++++ ...armak_clusters_snapshot_consul_restore.rst | 42 ++++ .../tarmak_clusters_snapshot_consul_save.rst | 42 ++++ .../tarmak/tarmak_clusters_snapshot_etcd.rst | 40 ++++ .../tarmak_clusters_snapshot_etcd_restore.rst | 45 ++++ .../tarmak_clusters_snapshot_etcd_save.rst | 42 ++++ docs/generated/reference/output/api-docs.html | 131 ++++++++++- docs/generated/reference/output/navData.js | 2 +- pkg/apis/tarmak/v1alpha1/types.go | 18 ++ .../tarmak/v1alpha1/zz_generated.deepcopy.go | 51 +++++ pkg/tarmak/snapshot/etcd/etcd.go | 205 +++++++++++++++++- pkg/tarmak/snapshot/snapshot.go | 2 + pkg/tarmak/utils/consts/consts.go | 4 + 19 files changed, 771 insertions(+), 18 deletions(-) create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst create mode 100644 docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst diff --git a/cmd/tarmak/cmd/cluster.go b/cmd/tarmak/cmd/cluster.go index e237b3b788..efc4e2f55a 100644 --- a/cmd/tarmak/cmd/cluster.go +++ b/cmd/tarmak/cmd/cluster.go @@ -115,6 +115,31 @@ func clusterKubeconfigFlags(fs *flag.FlagSet) { ) } +func clusterSnapshotEtcdRestoreFlags(fs *flag.FlagSet) { + store := &globalFlags.Cluster.Snapshot.Etcd.Restore + + fs.StringVar( + &store.K8sMain, + consts.RestoreK8sMainFlagName, + "", + "location of k8s-main snapshot backup", + ) + + fs.StringVar( + &store.K8sEvents, + consts.RestoreK8sEventsFlagName, + "", + "location of k8s-events snapshot backup", + ) + + fs.StringVar( + &store.Overlay, + consts.RestoreOverlayFlagName, + "", + "location of overlay snapshot backup", + ) +} + func init() { RootCmd.AddCommand(clusterCmd) } diff --git a/cmd/tarmak/cmd/cluster_apply.go b/cmd/tarmak/cmd/cluster_apply.go index 955b499532..8a68a7d92d 100644 --- a/cmd/tarmak/cmd/cluster_apply.go +++ b/cmd/tarmak/cmd/cluster_apply.go @@ -27,9 +27,7 @@ var clusterApplyCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - applyCmd := t.NewCmdTarmak(cmd.Flags(), args) - t.CancellationContext().WaitOrCancel(applyCmd.Apply) }, } diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go index 74cb1940eb..6454fca411 100644 --- a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go @@ -8,25 +8,35 @@ import ( "github.com/jetstack/tarmak/pkg/tarmak" "github.com/jetstack/tarmak/pkg/tarmak/snapshot/etcd" + "github.com/jetstack/tarmak/pkg/tarmak/utils/consts" ) var clusterSnapshotEtcdRestoreCmd = &cobra.Command{ - Use: "restore [source path]", - Short: "restore etcd cluster with source snapshot", + Use: "restore", + Short: "restore etcd cluster with source snapshots", PreRunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("expecting single target path, got=%d", len(args)) - } + if !cmd.Flags().Changed(consts.RestoreK8sMainFlagName) && + !cmd.Flags().Changed(consts.RestoreK8sEventsFlagName) && + !cmd.Flags().Changed(consts.RestoreOverlayFlagName) { + return fmt.Errorf("expecting at least one set flag of [%s %s %s]", + consts.RestoreK8sMainFlagName, + consts.RestoreK8sEventsFlagName, + consts.RestoreOverlayFlagName, + ) + } return nil }, Run: func(cmd *cobra.Command, args []string) { t := tarmak.New(globalFlags) - s := etcd.New(t, args[0]) + s := etcd.New(t, "") t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Restore) }, } func init() { + clusterSnapshotEtcdRestoreFlags( + clusterSnapshotEtcdRestoreCmd.PersistentFlags(), + ) clusterSnapshotEtcdCmd.AddCommand(clusterSnapshotEtcdRestoreCmd) } diff --git a/docs/cmd-docs.rst b/docs/cmd-docs.rst index ec1582433f..0a8e272ac9 100644 --- a/docs/cmd-docs.rst +++ b/docs/cmd-docs.rst @@ -94,6 +94,41 @@ Command line documentation for both tarmak and wing commands generated/cmd/tarmak/tarmak_clusters_set-current +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save + .. toctree:: :maxdepth: 1 diff --git a/docs/generated/cmd/tarmak/tarmak_clusters.rst b/docs/generated/cmd/tarmak/tarmak_clusters.rst index 3c392b100f..ba9f8e6954 100644 --- a/docs/generated/cmd/tarmak/tarmak_clusters.rst +++ b/docs/generated/cmd/tarmak/tarmak_clusters.rst @@ -47,5 +47,6 @@ SEE ALSO * `tarmak clusters list `_ - Print a list of clusters * `tarmak clusters plan `_ - Plan changes on the currently configured cluster * `tarmak clusters set-current `_ - Set current cluster in config +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters * `tarmak clusters ssh `_ - Log into an instance with SSH diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst new file mode 100644 index 0000000000..c224d2166b --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot: + +tarmak clusters snapshot +------------------------ + +Manage snapshots of remote consul and etcd clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots of remote consul and etcd clusters + +Options +~~~~~~~ + +:: + + -h, --help help for snapshot + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters `_ - Operations on clusters +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst new file mode 100644 index 0000000000..fee8445b27 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot_consul: + +tarmak clusters snapshot consul +------------------------------- + +Manage snapshots on remote consul clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots on remote consul clusters + +Options +~~~~~~~ + +:: + + -h, --help help for consul + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters +* `tarmak clusters snapshot consul restore `_ - restore consul cluster with source snapshot +* `tarmak clusters snapshot consul save `_ - save consul cluster snapshot to target path + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst new file mode 100644 index 0000000000..fa50a7867b --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_consul_restore: + +tarmak clusters snapshot consul restore +--------------------------------------- + +restore consul cluster with source snapshot + +Synopsis +~~~~~~~~ + + +restore consul cluster with source snapshot + +:: + + tarmak clusters snapshot consul restore [source path] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for restore + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst new file mode 100644 index 0000000000..fa51af35e6 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_consul_save: + +tarmak clusters snapshot consul save +------------------------------------ + +save consul cluster snapshot to target path + +Synopsis +~~~~~~~~ + + +save consul cluster snapshot to target path + +:: + + tarmak clusters snapshot consul save [target path] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for save + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst new file mode 100644 index 0000000000..44c53e8353 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot_etcd: + +tarmak clusters snapshot etcd +----------------------------- + +Manage snapshots on remote etcd clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots on remote etcd clusters + +Options +~~~~~~~ + +:: + + -h, --help help for etcd + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters +* `tarmak clusters snapshot etcd restore `_ - restore etcd cluster with source snapshots +* `tarmak clusters snapshot etcd save `_ - save etcd snapshot to target path prefix, i.e 'backup-' + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst new file mode 100644 index 0000000000..c476c97b3e --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst @@ -0,0 +1,45 @@ +.. _tarmak_clusters_snapshot_etcd_restore: + +tarmak clusters snapshot etcd restore +------------------------------------- + +restore etcd cluster with source snapshots + +Synopsis +~~~~~~~~ + + +restore etcd cluster with source snapshots + +:: + + tarmak clusters snapshot etcd restore [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for restore + --k8s-events string location of k8s-events snapshot backup + --k8s-main string location of k8s-main snapshot backup + --overlay string location of overlay snapshot backup + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst new file mode 100644 index 0000000000..fa549a2af6 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_etcd_save: + +tarmak clusters snapshot etcd save +---------------------------------- + +save etcd snapshot to target path prefix, i.e 'backup-' + +Synopsis +~~~~~~~~ + + +save etcd snapshot to target path prefix, i.e 'backup-' + +:: + + tarmak clusters snapshot etcd save [target path prefix] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for save + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --keep-containers do not clean-up terraform/packer containers after running them + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/reference/output/api-docs.html b/docs/generated/reference/output/api-docs.html index 1ba5b8dc0f..c78a0ada7d 100644 --- a/docs/generated/reference/output/api-docs.html +++ b/docs/generated/reference/output/api-docs.html @@ -11,7 +11,7 @@ - +

    Tarmak

    @@ -640,6 +640,10 @@

    ClusterFlags v1alpha1

    plan
    ClusterPlanFlags flags for handling images + +snapshot
    ClusterSnapshotFlags +flags for kubeconfig of clusters +

    ClusterImagesBuildFlags v1alpha1

    @@ -1377,6 +1381,131 @@

    ClusterPodSecurityPolicy v1alpha1

    +

    ClusterSnapshotEtcdFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotEtcdFlags
    +

    Contains the cluster snapshot etcd flags

    + + + + + + + + + + + + + + + +
    FieldDescription
    restore
    ClusterSnapshotEtcdRestoreFlags
    +

    ClusterSnapshotEtcdRestoreFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotEtcdRestoreFlags
    +

    Contains the cluster snapshot etcd restore flags

    + + + + + + + + + + + + + + + + + + + + + + + +
    FieldDescription
    k8sEvents
    string    		Path to k8s-main snapshot backup
    k8sMain
    string
    overlay
    string    		Path to k8s-events snapshot backup
    +

    ClusterSnapshotFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotFlags
    +

    Contains the cluster snapshot flags

    + + + + + + + + + + + + + + + +
    FieldDescription
    etcd
    ClusterSnapshotEtcdFlags

    EgressRule v1alpha1

    diff --git a/docs/generated/reference/output/navData.js b/docs/generated/reference/output/navData.js index 3d0d480511..662c0de380 100644 --- a/docs/generated/reference/output/navData.js +++ b/docs/generated/reference/output/navData.js @@ -1 +1 @@ -(function(){navData = {"toc":[{"section":"-strong-field-definitions-strong-","subsections":[{"section":"volume-v1alpha1"},{"section":"values-v1alpha1"},{"section":"taint-v1alpha1"},{"section":"subnet-v1alpha1"},{"section":"ssh-v1alpha1"},{"section":"providergcp-v1alpha1"},{"section":"providerazure-v1alpha1"},{"section":"provideramazon-v1alpha1"},{"section":"provider-v1alpha1"},{"section":"network-v1alpha1"},{"section":"loggingsinkelasticsearch-v1alpha1"},{"section":"loggingsink-v1alpha1"},{"section":"label-v1alpha1"},{"section":"kubernetesapi-v1alpha1"},{"section":"internetgw-v1alpha1"},{"section":"instancestatusmanifest-v1alpha1"},{"section":"instancespecmanifest-v1alpha1"},{"section":"instancepoolkubernetes-v1alpha1"},{"section":"instancepoolamazon-v1alpha1"},{"section":"instancepool-v1alpha1"},{"section":"ingressrule-v1alpha1"},{"section":"httpbasicauth-v1alpha1"},{"section":"firewall-v1alpha1"},{"section":"environment-v1alpha1"},{"section":"egressrule-v1alpha1"},{"section":"clusterpodsecuritypolicy-v1alpha1"},{"section":"clusterplanflags-v1alpha1"},{"section":"clusterkubernetestiller-v1alpha1"},{"section":"clusterkubernetesprometheus-v1alpha1"},{"section":"clusterkubernetesdashboard-v1alpha1"},{"section":"clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1"},{"section":"clusterkubernetesclusterautoscaler-v1alpha1"},{"section":"clusterkubernetesapiserveroidc-v1alpha1"},{"section":"clusterkubernetesapiserveramazonaccesslogs-v1alpha1"},{"section":"clusterkubernetesapiserveramazon-v1alpha1"},{"section":"clusterkubernetesapiserver-v1alpha1"},{"section":"clusterkubernetes-v1alpha1"},{"section":"clusterkubeconfigflags-v1alpha1"},{"section":"clusterimagesflags-v1alpha1"},{"section":"clusterimagesbuildflags-v1alpha1"},{"section":"clusterflags-v1alpha1"},{"section":"clusterdestroyflags-v1alpha1"},{"section":"clusterapplyflags-v1alpha1"},{"section":"clusteramazon-v1alpha1"},{"section":"amazonesproxy-v1alpha1"}]},{"section":"-strong-old-api-versions-strong-","subsections":[]},{"section":"instance-v1alpha1","subsections":[]},{"section":"cluster-v1alpha1","subsections":[]},{"section":"flags-v1alpha1","subsections":[]},{"section":"image-v1alpha1","subsections":[]},{"section":"config-v1alpha1","subsections":[]},{"section":"-strong-tarmak-strong-","subsections":[]}],"flatToc":["volume-v1alpha1","values-v1alpha1","taint-v1alpha1","subnet-v1alpha1","ssh-v1alpha1","providergcp-v1alpha1","providerazure-v1alpha1","provideramazon-v1alpha1","provider-v1alpha1","network-v1alpha1","loggingsinkelasticsearch-v1alpha1","loggingsink-v1alpha1","label-v1alpha1","kubernetesapi-v1alpha1","internetgw-v1alpha1","instancestatusmanifest-v1alpha1","instancespecmanifest-v1alpha1","instancepoolkubernetes-v1alpha1","instancepoolamazon-v1alpha1","instancepool-v1alpha1","ingressrule-v1alpha1","httpbasicauth-v1alpha1","firewall-v1alpha1","environment-v1alpha1","egressrule-v1alpha1","clusterpodsecuritypolicy-v1alpha1","clusterplanflags-v1alpha1","clusterkubernetestiller-v1alpha1","clusterkubernetesprometheus-v1alpha1","clusterkubernetesdashboard-v1alpha1","clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1","clusterkubernetesclusterautoscaler-v1alpha1","clusterkubernetesapiserveroidc-v1alpha1","clusterkubernetesapiserveramazonaccesslogs-v1alpha1","clusterkubernetesapiserveramazon-v1alpha1","clusterkubernetesapiserver-v1alpha1","clusterkubernetes-v1alpha1","clusterkubeconfigflags-v1alpha1","clusterimagesflags-v1alpha1","clusterimagesbuildflags-v1alpha1","clusterflags-v1alpha1","clusterdestroyflags-v1alpha1","clusterapplyflags-v1alpha1","clusteramazon-v1alpha1","amazonesproxy-v1alpha1","-strong-field-definitions-strong-","-strong-old-api-versions-strong-","instance-v1alpha1","cluster-v1alpha1","flags-v1alpha1","image-v1alpha1","config-v1alpha1","-strong-tarmak-strong-"]};})(); \ No newline at end of file +(function(){navData = {"toc":[{"section":"-strong-field-definitions-strong-","subsections":[{"section":"volume-v1alpha1"},{"section":"values-v1alpha1"},{"section":"taint-v1alpha1"},{"section":"subnet-v1alpha1"},{"section":"ssh-v1alpha1"},{"section":"providergcp-v1alpha1"},{"section":"providerazure-v1alpha1"},{"section":"provideramazon-v1alpha1"},{"section":"provider-v1alpha1"},{"section":"network-v1alpha1"},{"section":"loggingsinkelasticsearch-v1alpha1"},{"section":"loggingsink-v1alpha1"},{"section":"label-v1alpha1"},{"section":"kubernetesapi-v1alpha1"},{"section":"internetgw-v1alpha1"},{"section":"instancestatusmanifest-v1alpha1"},{"section":"instancespecmanifest-v1alpha1"},{"section":"instancepoolkubernetes-v1alpha1"},{"section":"instancepoolamazon-v1alpha1"},{"section":"instancepool-v1alpha1"},{"section":"ingressrule-v1alpha1"},{"section":"httpbasicauth-v1alpha1"},{"section":"firewall-v1alpha1"},{"section":"environment-v1alpha1"},{"section":"egressrule-v1alpha1"},{"section":"clustersnapshotflags-v1alpha1"},{"section":"clustersnapshotetcdrestoreflags-v1alpha1"},{"section":"clustersnapshotetcdflags-v1alpha1"},{"section":"clusterpodsecuritypolicy-v1alpha1"},{"section":"clusterplanflags-v1alpha1"},{"section":"clusterkubernetestiller-v1alpha1"},{"section":"clusterkubernetesprometheus-v1alpha1"},{"section":"clusterkubernetesdashboard-v1alpha1"},{"section":"clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1"},{"section":"clusterkubernetesclusterautoscaler-v1alpha1"},{"section":"clusterkubernetesapiserveroidc-v1alpha1"},{"section":"clusterkubernetesapiserveramazonaccesslogs-v1alpha1"},{"section":"clusterkubernetesapiserveramazon-v1alpha1"},{"section":"clusterkubernetesapiserver-v1alpha1"},{"section":"clusterkubernetes-v1alpha1"},{"section":"clusterkubeconfigflags-v1alpha1"},{"section":"clusterimagesflags-v1alpha1"},{"section":"clusterimagesbuildflags-v1alpha1"},{"section":"clusterflags-v1alpha1"},{"section":"clusterdestroyflags-v1alpha1"},{"section":"clusterapplyflags-v1alpha1"},{"section":"clusteramazon-v1alpha1"},{"section":"amazonesproxy-v1alpha1"}]},{"section":"-strong-old-api-versions-strong-","subsections":[]},{"section":"instance-v1alpha1","subsections":[]},{"section":"cluster-v1alpha1","subsections":[]},{"section":"flags-v1alpha1","subsections":[]},{"section":"image-v1alpha1","subsections":[]},{"section":"config-v1alpha1","subsections":[]},{"section":"-strong-tarmak-strong-","subsections":[]}],"flatToc":["volume-v1alpha1","values-v1alpha1","taint-v1alpha1","subnet-v1alpha1","ssh-v1alpha1","providergcp-v1alpha1","providerazure-v1alpha1","provideramazon-v1alpha1","provider-v1alpha1","network-v1alpha1","loggingsinkelasticsearch-v1alpha1","loggingsink-v1alpha1","label-v1alpha1","kubernetesapi-v1alpha1","internetgw-v1alpha1","instancestatusmanifest-v1alpha1","instancespecmanifest-v1alpha1","instancepoolkubernetes-v1alpha1","instancepoolamazon-v1alpha1","instancepool-v1alpha1","ingressrule-v1alpha1","httpbasicauth-v1alpha1","firewall-v1alpha1","environment-v1alpha1","egressrule-v1alpha1","clustersnapshotflags-v1alpha1","clustersnapshotetcdrestoreflags-v1alpha1","clustersnapshotetcdflags-v1alpha1","clusterpodsecuritypolicy-v1alpha1","clusterplanflags-v1alpha1","clusterkubernetestiller-v1alpha1","clusterkubernetesprometheus-v1alpha1","clusterkubernetesdashboard-v1alpha1","clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1","clusterkubernetesclusterautoscaler-v1alpha1","clusterkubernetesapiserveroidc-v1alpha1","clusterkubernetesapiserveramazonaccesslogs-v1alpha1","clusterkubernetesapiserveramazon-v1alpha1","clusterkubernetesapiserver-v1alpha1","clusterkubernetes-v1alpha1","clusterkubeconfigflags-v1alpha1","clusterimagesflags-v1alpha1","clusterimagesbuildflags-v1alpha1","clusterflags-v1alpha1","clusterdestroyflags-v1alpha1","clusterapplyflags-v1alpha1","clusteramazon-v1alpha1","amazonesproxy-v1alpha1","-strong-field-definitions-strong-","-strong-old-api-versions-strong-","instance-v1alpha1","cluster-v1alpha1","flags-v1alpha1","image-v1alpha1","config-v1alpha1","-strong-tarmak-strong-"]};})(); \ No newline at end of file diff --git a/pkg/apis/tarmak/v1alpha1/types.go b/pkg/apis/tarmak/v1alpha1/types.go index 7bc0683d78..fa769351c2 100644 --- a/pkg/apis/tarmak/v1alpha1/types.go +++ b/pkg/apis/tarmak/v1alpha1/types.go @@ -146,6 +146,7 @@ type ClusterFlags struct { Images ClusterImagesFlags `json:"images,omitempty"` // flags for handling images Plan ClusterPlanFlags `json:"plan,omitempty"` // flags for planning clusters Kubeconfig ClusterKubeconfigFlags `json:"kubeconfig,omitempty"` // flags for kubeconfig of clusters + Snapshot ClusterSnapshotFlags `json:"snapshot,omitempty"` // flags for snapshots of clusters } // Contains the cluster plan flags @@ -187,3 +188,20 @@ type ClusterImagesBuildFlags struct { type ClusterKubeconfigFlags struct { Path string `json:"path,omitempty"` // Path to save kubeconfig to } + +// Contains the cluster snapshot flags +type ClusterSnapshotFlags struct { + Etcd ClusterSnapshotEtcdFlags `json:"etcd,omitempty"` // flags for handling etcd snapshots +} + +// Contains the cluster snapshot etcd flags +type ClusterSnapshotEtcdFlags struct { + Restore ClusterSnapshotEtcdRestoreFlags `json:"restore,omitempty"` // flags for handling etcd snapshot restore +} + +// Contains the cluster snapshot etcd restore flags +type ClusterSnapshotEtcdRestoreFlags struct { + K8sMain string `json:"k8sMain,omitempty"` // Path to k8s-main snapshot backup + K8sEvents string `json:"k8sEvents,omitempty"` // Path to k8s-events snapshot backup + Overlay string `json:"overlay,omitempty"` // Path to overlay snapshot backup +} diff --git a/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go index aee63e81cd..7e2bfc809a 100644 --- a/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,7 @@ func (in *ClusterFlags) DeepCopyInto(out *ClusterFlags) { out.Images = in.Images out.Plan = in.Plan out.Kubeconfig = in.Kubeconfig + out.Snapshot = in.Snapshot return } @@ -129,6 +130,56 @@ func (in *ClusterPlanFlags) DeepCopy() *ClusterPlanFlags { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotEtcdFlags) DeepCopyInto(out *ClusterSnapshotEtcdFlags) { + *out = *in + out.Restore = in.Restore + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotEtcdFlags. +func (in *ClusterSnapshotEtcdFlags) DeepCopy() *ClusterSnapshotEtcdFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotEtcdFlags) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotEtcdRestoreFlags) DeepCopyInto(out *ClusterSnapshotEtcdRestoreFlags) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotEtcdRestoreFlags. +func (in *ClusterSnapshotEtcdRestoreFlags) DeepCopy() *ClusterSnapshotEtcdRestoreFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotEtcdRestoreFlags) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotFlags) DeepCopyInto(out *ClusterSnapshotFlags) { + *out = *in + out.Etcd = in.Etcd + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotFlags. +func (in *ClusterSnapshotFlags) DeepCopy() *ClusterSnapshotFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotFlags) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Config) DeepCopyInto(out *Config) { *out = *in diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go index 3bc86b87f3..54b6e35779 100644 --- a/pkg/tarmak/snapshot/etcd/etcd.go +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -2,7 +2,8 @@ package etcd import ( - //"bufio" + "crypto/rand" + "encoding/base64" "fmt" "io" "strings" @@ -15,6 +16,7 @@ import ( clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" "github.com/jetstack/tarmak/pkg/tarmak/interfaces" "github.com/jetstack/tarmak/pkg/tarmak/snapshot" + "github.com/jetstack/tarmak/pkg/tarmak/utils/consts" ) var _ interfaces.Snapshot = &Etcd{} @@ -25,9 +27,9 @@ const ( var ( stores = []map[string]string{ - {"store": "k8s-main", "file": "k8s", "port": "2379"}, - {"store": "k8s-events", "file": "k8s", "port": "2369"}, - {"store": "overlay", "file": "overlay", "port": "2359"}, + {"cluster": consts.RestoreK8sMainFlagName, "file": "k8s", "client_port": "2379", "peer_port": "2380"}, + {"cluster": consts.RestoreK8sEventsFlagName, "file": "k8s", "client_port": "2369", "peer_port": "2370"}, + {"cluster": consts.RestoreOverlayFlagName, "file": "overlay", "client_port": "2359", "peer_port": "2360"}, } envCmd = ` @@ -36,7 +38,7 @@ export ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem; export ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem; export ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem; export ETCDCTL_API=3; -export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{port}}; +export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{client_port}}; ` ) @@ -77,7 +79,7 @@ func (e *Etcd) Save() error { defer wg.Done() hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", - store["store"], time.Now().Format(snapshot.TimeLayout)) + store["cluster"], time.Now().Format(snapshot.TimeLayout)) cmdArgs := fmt.Sprintf(`sudo /bin/bash -c "%s %s"`, e.template(envCmd, store), fmt.Sprintf(etcdctlCmd, "save", hostPath)) @@ -91,7 +93,7 @@ func (e *Etcd) Save() error { return } - targetPath := fmt.Sprintf("%s%s.db", e.path, store["store"]) + targetPath := fmt.Sprintf("%s%s.db", e.path, store["cluster"]) reader, writer := io.Pipe() err = snapshot.TarFromStream(func() error { err := snapshot.SSHCmd(e, aliases[0], fmt.Sprintf(snapshot.GZipCCmd, hostPath), @@ -107,7 +109,7 @@ func (e *Etcd) Save() error { return } - e.log.Infof("etcd %s snapshot saved to %s", store["store"], targetPath) + e.log.Infof("etcd %s snapshot saved to %s", store["cluster"], targetPath) select { case <-e.ctx.Done(): @@ -139,9 +141,171 @@ func (e *Etcd) Restore() error { } e.aliases = aliases + restoreFunc := func(host, path, token string, store map[string]string) error { + reader, writer := io.Pipe() + hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", + store["cluster"], time.Now().Format(snapshot.TimeLayout)) + + err = snapshot.TarToStream(func() error { + err := snapshot.SSHCmd(e, host, fmt.Sprintf(snapshot.GZipDCmd, hostPath), reader, nil, nil) + return err + }, writer, path) + if err != nil { + return err + } + + cmdArgs := fmt.Sprintf(`set -e; +sudo systemctl stop etcd-%s +`, store["cluster"]) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + cmdArgs = e.template(`set -e; +sudo mkdir -p /var/lib/etcd_backup; +sudo rsync -a --delete --ignore-missing-args /var/lib/etcd/{{cluster}} /var/lib/etcd_backup/; +sudo rm -rf /var/lib/etcd/{{cluster}}; +`, store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + initialCluster := e.initialClusterString(host, store) + for _, a := range aliases[1:] { + initialCluster = strings.Join( + []string{initialCluster, e.initialClusterString(a, store)}, ",", + ) + } + + cmdArgs = e.template(fmt.Sprintf(`set -e; +sudo ETCDCTL_API=3 /opt/bin/etcdctl snapshot restore %s \ +--name=%s.%s.%s \ +--data-dir=/var/lib/etcd/{{cluster}} \ +--initial-advertise-peer-urls=https://%s.%s.%s:{{peer_port}} \ +--initial-cluster=%s \ +--initial-cluster-token=etcd-{{cluster}}-%s +`, + hostPath, + host, e.clusterName(), e.privateZone(), + host, e.clusterName(), e.privateZone(), + initialCluster, + token, + ), store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + cmdArgs = e.template(`set -e; + sudo chown -R etcd:etcd /var/lib/etcd/{{cluster}} + `, store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + return nil + } + + startEtcdFunc := func(host string, store map[string]string) error { + cmdArgs := e.template(`set -e; +sudo systemctl start etcd-{{cluster}} + `, store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + return nil + } + + healthCheckFunc := func(host string, store map[string]string) error { + endpoints := e.endpointsString(host, store) + for _, a := range aliases[1:] { + endpoints = strings.Join( + []string{endpoints, e.endpointsString(a, store)}, ",", + ) + } + + cmdArgs := e.template(fmt.Sprint(`%s +sudo /opt/bin/etcdctl endpoint health +--endpoints=%s + `, envCmd, endpoints), store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + return nil + } + + for _, store := range stores { + value := e.restoreFlagValue(store["cluster"]) + if value == "" { + continue + } + + b := make([]byte, 32) + _, err := rand.Read(b) + if err != nil { + return fmt.Errorf("failed to create random etcd initial token: %s", err) + } + token := base64.URLEncoding.EncodeToString(b) + + for _, a := range aliases { + e.log.Infof("restoring etcd %s on host %s", store["cluster"], a) + err := restoreFunc(a, value, token, store) + if err != nil { + return err + } + } + + for _, a := range aliases { + e.log.Infof("starting etcd %s on host %s", store["cluster"], a) + err := startEtcdFunc(a, store) + if err != nil { + return err + } + } + + for _, a := range aliases { + e.log.Infof("checking health of etcd %s on host %s", store["cluster"], a) + err := healthCheckFunc(a, store) + if err != nil { + return err + } + } + + e.log.Infof("successfully restored etcd cluster %s with snapshot %s", store["cluster"], value) + } + + e.log.Info("restarting API servers on master hosts") + masters, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeMaster) + for _, master := range masters { + cmdArgs := " sudo systemctl restart kube-apiserver" + err = snapshot.SSHCmd(e, master, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + } + return nil } +func (e *Etcd) initialClusterString(host string, store map[string]string) string { + return fmt.Sprintf("%s.%s.%s=https://%s.%s.%s:%s", + host, e.clusterName(), e.privateZone(), + host, e.clusterName(), e.privateZone(), store["peer_port"]) +} + +func (e *Etcd) endpointsString(host string, store map[string]string) string { + return fmt.Sprintf("%s.%s.%s=https://%s.%s.%s:%s", + host, e.clusterName(), e.privateZone(), + host, e.clusterName(), e.privateZone(), store["client_port"]) +} + func (e *Etcd) template(args string, vars map[string]string) string { for k, v := range vars { args = strings.Replace(args, fmt.Sprintf("{{%s}}", k), v, -1) @@ -157,3 +321,28 @@ func (e *Etcd) Log() *logrus.Entry { func (e *Etcd) SSH() interfaces.SSH { return e.ssh } + +func (e *Etcd) clusterName() string { + return e.tarmak.Cluster().ClusterName() +} + +func (e *Etcd) privateZone() string { + return e.tarmak.Environment().Config().PrivateZone +} + +func (e *Etcd) restoreFlagValue(flag string) string { + rf := e.tarmak.ClusterFlags().Snapshot.Etcd.Restore + for _, db := range []struct { + name, value string + }{ + {consts.RestoreK8sMainFlagName, rf.K8sMain}, + {consts.RestoreK8sEventsFlagName, rf.K8sEvents}, + {consts.RestoreOverlayFlagName, rf.Overlay}, + } { + if db.name == flag { + return db.value + } + } + + return "" +} diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go index 7888a36fee..c0a72c30a1 100644 --- a/pkg/tarmak/snapshot/snapshot.go +++ b/pkg/tarmak/snapshot/snapshot.go @@ -159,6 +159,8 @@ func TarToStream(sshCmd func() error, stream io.WriteCloser, src string) error { } func SSHCmd(s interfaces.Snapshot, host, cmd string, stdin io.Reader, stdout, stderr io.Writer) error { + fmt.Printf("$ %s\n", cmd) + for _, w := range []struct { writer io.Writer out string diff --git a/pkg/tarmak/utils/consts/consts.go b/pkg/tarmak/utils/consts/consts.go index 98c07253f4..4ca16fb8d1 100644 --- a/pkg/tarmak/utils/consts/consts.go +++ b/pkg/tarmak/utils/consts/consts.go @@ -9,4 +9,8 @@ const ( DefaultKubeconfigPath = "${TARMAK_CONFIG}/${CURRENT_CLUSTER}/kubeconfig" KubeconfigFlagName = "public-api-endpoint" + + RestoreK8sMainFlagName = "k8s-main" + RestoreK8sEventsFlagName = "k8s-events" + RestoreOverlayFlagName = "overlay" )