[content-service] Fix backup restore

components/content-service/go.mod

@@ -25,13 +25,13 @@ require (
 	google.golang.org/protobuf v1.28.0
 )
 
-require github.com/containers/storage v1.39.0
+require golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 
 require (
 	cloud.google.com/go v0.83.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -42,36 +42,29 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jstemmer/go-junit-report v0.9.1 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect
 	github.com/minio/sha256-simd v0.1.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/sys/mountinfo v0.6.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/opencontainers/runc v1.1.0 // indirect
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rs/xid v1.2.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/stretchr/testify v1.7.1 // indirect
 	github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.8.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff // indirect

components/content-service/pkg/archive/tar.go

@@ -5,13 +5,19 @@
 package archive
 
 import (
+	"archive/tar"
 	"context"
 	"io"
+	"os"
+	"os/exec"
+	"path"
+	"sort"
+	"syscall"
 	"time"
 
-	"github.com/containers/storage/pkg/archive"
-	"github.com/containers/storage/pkg/idtools"
 	"github.com/opentracing/opentracing-go"
+	"golang.org/x/sys/unix"
+	"golang.org/x/xerrors"
 
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
@@ -49,6 +55,12 @@ func WithGIDMapping(mappings []IDMapping) TarOption {
 
 // ExtractTarbal extracts an OCI compatible tar file src to the folder dst, expecting the overlay whiteout format
 func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOption) (err error) {
+	type Info struct {
+		UID, GID  int
+		IsSymlink bool
+		Xattrs    map[string]string
+	}
+
 	//nolint:staticcheck,ineffassign
 	span, ctx := opentracing.StartSpanFromContext(ctx, "extractTarbal")
 	span.LogKV("dst", dst)
@@ -60,29 +72,136 @@ func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOp
 		opt(&cfg)
 	}
 
-	uidMaps := make([]idtools.IDMap, len(cfg.UIDMaps))
-	for i, m := range cfg.UIDMaps {
-		uidMaps[i] = idtools.IDMap{
-			ContainerID: m.ContainerID,
-			HostID:      m.HostID,
-			Size:        m.Size,
+	pipeReader, pipeWriter := io.Pipe()
+	teeReader := io.TeeReader(src, pipeWriter)
+
+	tarReader := tar.NewReader(pipeReader)
+
+	finished := make(chan bool)
+	m := make(map[string]Info)
+
+	go func() {
+		defer close(finished)
+		for {
+			hdr, err := tarReader.Next()
+			if err == io.EOF {
+				finished <- true
+				return
+			}
+
+			if err != nil {
+				log.WithError(err).Error("error reading tar")
+				return
+			}
+
+			m[hdr.Name] = Info{
+				UID:       hdr.Uid,
+				GID:       hdr.Gid,
+				IsSymlink: (hdr.Linkname != ""),
+				//nolint:staticcheck
+				Xattrs: hdr.Xattrs,
+			}
 		}
+	}()
+
+	// Be explicit about the tar flags. We want to restore the exact content without changes
+	tarcmd := exec.Command(
+		"tar",
+		"--extract",
+		"--preserve-permissions",
+		"--xattrs", "--xattrs-include=security.capability",
+	)
+	tarcmd.Dir = dst
+	tarcmd.Stdin = teeReader
+
+	var msg []byte
+	msg, err = tarcmd.CombinedOutput()
+	if err != nil {
+		return xerrors.Errorf("tar %s: %s", dst, err.Error()+";"+string(msg))
 	}
-	gidMaps := make([]idtools.IDMap, len(cfg.GIDMaps))
-	for i, m := range cfg.GIDMaps {
-		gidMaps[i] = idtools.IDMap{
-			ContainerID: m.ContainerID,
-			HostID:      m.HostID,
-			Size:        m.Size,
-		}
+
+	log.WithField("log", string(msg)).Debug("decompressing tar stream log")
+
+	<-finished
+
+	// lets create a sorted list of pathes and chown depth first.
+	paths := make([]string, 0, len(m))
+	for path := range m {
+		paths = append(paths, path)
 	}
+	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
 
-	err = archive.Untar(src, dst, &archive.TarOptions{
-		UIDMaps:     uidMaps,
-		GIDMaps:     gidMaps,
-		Compression: archive.Uncompressed,
-	})
+	// We need to remap the UID and GID between the host and the container to avoid permission issues.
+	for _, p := range paths {
+		v := m[p]
+		uid := toHostID(v.UID, cfg.UIDMaps)
+		gid := toHostID(v.GID, cfg.GIDMaps)
+
+		if v.IsSymlink {
+			continue
+		}
+
+		err = remapFile(path.Join(dst, p), uid, gid, v.Xattrs)
+		if err != nil {
+			log.WithError(err).WithField("uid", uid).WithField("gid", gid).WithField("path", p).Warn("cannot chown")
+		}
+	}
 
 	log.WithField("duration", time.Since(start).Milliseconds()).Debug("untar complete")
-	return
+	return nil
+}
+
+func toHostID(containerID int, idMap []IDMapping) int {
+	for _, m := range idMap {
+		if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
+			hostID := m.HostID + (containerID - m.ContainerID)
+			return hostID
+		}
+	}
+	return containerID
+}
+
+// remapFile changes the UID and GID of a file preserving existing file mode bits.
+func remapFile(name string, uid, gid int, xattrs map[string]string) error {
+	// current info of the file before any change
+	fileInfo, err := os.Stat(name)
+	if err != nil {
+		return err
+	}
+
+	// nothing to do for symlinks
+	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
+		return nil
+	}
+
+	// changing UID or GID can break files with suid/sgid
+	err = os.Lchown(name, uid, gid)
+	if err != nil {
+		return err
+	}
+
+	// restore original permissions
+	err = os.Chmod(name, fileInfo.Mode())
+	if err != nil {
+		return err
+	}
+
+	for key, value := range xattrs {
+		if err := unix.Lsetxattr(name, key, []byte(value), 0); err != nil {
+			log.WithField("name", key).WithField("value", value).WithField("file", name).WithError(err).Error("restoring extended attributes")
+			if err == syscall.ENOTSUP || err == syscall.EPERM {
+				continue
+			}
+
+			return err
+		}
+	}
+
+	// restore file times
+	fileTime := fileInfo.Sys().(*syscall.Stat_t)
+	return os.Chtimes(name, timespecToTime(fileTime.Atim), timespecToTime(fileTime.Mtim))
+}
+
+func timespecToTime(ts syscall.Timespec) time.Time {
+	return time.Unix(int64(ts.Sec), int64(ts.Nsec))
 }

components/public-api-server/go.mod

@@ -5,8 +5,10 @@ go 1.17
 require (
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/public-api v0.0.0-00010101000000-000000000000
+	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.28.0
 )
 
 require (
@@ -25,14 +27,12 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
 	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 

components/public-api/go/go.mod

@@ -3,7 +3,6 @@ module github.com/gitpod-io/gitpod/public-api
 go 1.18
 
 require (
-	google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154
 	google.golang.org/grpc v1.45.0
 	google.golang.org/protobuf v1.28.0
 )
@@ -14,6 +13,7 @@ require (
 	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
 )
 
 replace github.com/gitpod-io/gitpod/content-service/api => ../../content-service-api/go // leeway

components/supervisor/go.mod

@@ -44,8 +44,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/containers/storage v1.39.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/go-ozzo/ozzo-validation v3.5.0+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -58,19 +56,18 @@ require (
 	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect
 	github.com/minio/minio-go/v7 v7.0.11 // indirect
 	github.com/minio/sha256-simd v0.1.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/sys/mountinfo v0.6.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/opencontainers/runc v1.1.0 // indirect
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
@@ -80,10 +77,8 @@ require (
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
 	github.com/sourcegraph/jsonrpc2 v0.0.0-20200429184054-15c2290dcb37 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.8.0 // indirect
@@ -96,6 +91,7 @@ require (
 	google.golang.org/api v0.48.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/segmentio/analytics-go.v3 v3.1.0 // indirect
 )