diff --git a/cmd/keytransparency-monitor/Dockerfile b/cmd/keytransparency-monitor/Dockerfile new file mode 100644 index 000000000..7754fa608 --- /dev/null +++ b/cmd/keytransparency-monitor/Dockerfile @@ -0,0 +1,12 @@ +FROM golang + +ADD keytransparency/genfiles/* /kt/ +ADD ./keytransparency /go/src/github.com/google/keytransparency +ADD ./trillian /go/src/github.com/google/trillian +WORKDIR /go/src/github.com/google/keytransparency + +RUN go get -tags="mysql" ./cmd/keytransparency-monitor + +ENTRYPOINT ["/go/bin/keytransparency-monitor"] + +EXPOSE 8099 diff --git a/cmd/keytransparency-monitor/main.go b/cmd/keytransparency-monitor/main.go new file mode 100644 index 000000000..b76685507 --- /dev/null +++ b/cmd/keytransparency-monitor/main.go @@ -0,0 +1,228 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "flag" + "net" + "net/http" + "strings" + "time" + + "github.com/golang/glog" + "github.com/google/keytransparency/impl/monitor" + "github.com/google/trillian" + "github.com/google/trillian/crypto" + "github.com/google/trillian/crypto/keys/pem" + "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "golang.org/x/net/context" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/reflection" + + cmon "github.com/google/keytransparency/core/monitor" + "github.com/google/keytransparency/core/monitor/storage" + kpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + "github.com/google/keytransparency/impl/monitor/client" + spb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service" + mopb "github.com/google/keytransparency/impl/proto/monitor_v1_service" + mupb "github.com/google/keytransparency/impl/proto/mutation_v1_service" + _ "github.com/google/trillian/merkle/coniks" // Register coniks + _ "github.com/google/trillian/merkle/objhasher" // Register objhasher +) + +var ( + addr = flag.String("addr", ":8099", "The ip:port combination to listen on") + keyFile = flag.String("tls-key", "genfiles/server.key", "TLS private key file") + certFile = flag.String("tls-cert", "genfiles/server.pem", "TLS cert file") + + signingKey = flag.String("sign-key", "genfiles/monitor_sign-key.pem", "Path to private key PEM for SMH signing") + signingKeyPassword = flag.String("password", "towel", "Password of the private key PEM file for SMH signing") + ktURL = flag.String("kt-url", "localhost:8080", "URL of key-server.") + insecure = flag.Bool("insecure", false, "Skip TLS checks") + ktCert = flag.String("kt-cert", "genfiles/server.crt", "Path to kt-server's public key") + + pollPeriod = flag.Duration("poll-period", time.Second*5, "Maximum time between polling the key-server. Ideally, this is equal to the min-period of paramerter of the keyserver.") + + // TODO(ismail): expose prometheus metrics: a variable that tracks valid/invalid MHs + // metricsAddr = flag.String("metrics-addr", ":8081", "The ip:port to publish metrics on") +) + +func grpcGatewayMux(addr string) (*runtime.ServeMux, error) { + ctx := context.Background() + creds, err := credentials.NewClientTLSFromFile(*certFile, "") + if err != nil { + return nil, err + } + dopts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + gwmux := runtime.NewServeMux() + if err := mopb.RegisterMonitorServiceHandlerFromEndpoint(ctx, gwmux, addr, dopts); err != nil { + return nil, err + } + + return gwmux, nil +} + +// grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC +// connections or otherHandler otherwise. Copied from cockroachdb. +func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // This is a partial recreation of gRPC's internal checks. + // https://github.com/grpc/grpc-go/blob/master/transport/handler_server.go#L62 + if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + } else { + otherHandler.ServeHTTP(w, r) + } + }) +} + +func main() { + flag.Parse() + + creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) + if err != nil { + glog.Exitf("Failed to load server credentials %v", err) + } + + // Create gRPC server. + grpcServer := grpc.NewServer( + grpc.Creds(creds), + grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), + grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), + ) + + // Connect to the kt-server's mutation API: + grpcc, err := dial() + if err != nil { + glog.Fatalf("Error Dialing %v: %v", ktURL, err) + } + mcc := mupb.NewMutationServiceClient(grpcc) + + // Read signing key: + key, err := pem.ReadPrivateKeyFile(*signingKey, *signingKeyPassword) + if err != nil { + glog.Fatalf("Could not create signer from %v: %v", *signingKey, err) + } + ctx := context.Background() + logTree, mapTree, err := getTrees(ctx, grpcc) + if err != nil { + glog.Fatalf("Could not read domain info %v:", err) + } + + store := storage.New() + srv := monitor.New(store) + mopb.RegisterMonitorServiceServer(grpcServer, srv) + reflection.Register(grpcServer) + grpc_prometheus.Register(grpcServer) + grpc_prometheus.EnableHandlingTimeHistogram() + + // Create HTTP handlers and gRPC gateway. + gwmux, err := grpcGatewayMux(*addr) + if err != nil { + glog.Exitf("Failed setting up REST proxy: %v", err) + } + + // Insert handlers for other http paths here. + mux := http.NewServeMux() + mux.Handle("/", gwmux) + + // initialize the mutations API client and feed the responses it got + // into the monitor: + mon, err := cmon.New(logTree, mapTree, crypto.NewSHA256Signer(key), store) + if err != nil { + glog.Exitf("Failed to initialize monitor: %v", err) + } + mutCli := client.New(mcc, *pollPeriod) + responses, errs := mutCli.StartPolling(1) + go func() { + for { + select { + case mutResp := <-responses: + glog.Infof("Received mutations response: %v", mutResp.Epoch) + if err := mon.Process(mutResp); err != nil { + glog.Infof("Error processing mutations response: %v", err) + } + case err := <-errs: + // this is OK if there were no mutations in between: + // TODO(ismail): handle the case when the known maxDuration has + // passed and no epoch was issued? + glog.Infof("Could not retrieve mutations API response %v", err) + } + } + }() + + // Serve HTTP2 server over TLS. + glog.Infof("Listening on %v", *addr) + if err := http.ListenAndServeTLS(*addr, *certFile, *keyFile, + grpcHandlerFunc(grpcServer, mux)); err != nil { + glog.Errorf("ListenAndServeTLS: %v", err) + } +} + +func dial() (*grpc.ClientConn, error) { + var opts []grpc.DialOption + + transportCreds, err := transportCreds(*ktURL, *ktCert, *insecure) + if err != nil { + return nil, err + } + opts = append(opts, grpc.WithTransportCredentials(transportCreds)) + + // TODO(ismail): authenticate the monitor to the kt-server: + cc, err := grpc.Dial(*ktURL, opts...) + if err != nil { + return nil, err + } + return cc, nil +} + +// TODO(ismail): refactor client and monitor to use the same methods +func transportCreds(ktURL string, ktCert string, insecure bool) (credentials.TransportCredentials, error) { + // copied from keytransparency-client/cmd/root.go: transportCreds + host, _, err := net.SplitHostPort(ktURL) + if err != nil { + return nil, err + } + + switch { + case insecure: // Impatient insecure. + return credentials.NewTLS(&tls.Config{ + InsecureSkipVerify: true, + }), nil + + case ktCert != "": // Custom CA Cert. + return credentials.NewClientTLSFromFile(ktCert, host) + + default: // Use the local set of root certs. + return credentials.NewClientTLSFromCert(nil, host), nil + } +} + +// config selects a source for and returns the client configuration. +func getTrees(ctx context.Context, cc *grpc.ClientConn) (logTree *trillian.Tree, mapTree *trillian.Tree, err error) { + ktClient := spb.NewKeyTransparencyServiceClient(cc) + resp, err2 := ktClient.GetDomainInfo(ctx, &kpb.GetDomainInfoRequest{}) + if err2 != nil { + err = err2 + return + } + logTree = resp.GetLog() + mapTree = resp.GetMap() + return +} diff --git a/core/monitor/monitor.go b/core/monitor/monitor.go new file mode 100644 index 000000000..419936b28 --- /dev/null +++ b/core/monitor/monitor.go @@ -0,0 +1,87 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitor + +import ( + "crypto" + "fmt" + "time" + + "github.com/golang/glog" + + "github.com/google/keytransparency/core/monitor/storage" + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + + "github.com/google/trillian" + tcrypto "github.com/google/trillian/crypto" + "github.com/google/trillian/merkle" + "github.com/google/trillian/merkle/hashers" +) + +// Monitor holds the internal state for a monitor accessing the mutations API +// and for verifying its responses. +type Monitor struct { + hasher hashers.MapHasher + logPubKey crypto.PublicKey + mapPubKey crypto.PublicKey + logVerifier merkle.LogVerifier + signer *tcrypto.Signer + // TODO(ismail): update last trusted signed log root + //trusted trillian.SignedLogRoot + store *storage.Storage +} + +// New creates a new instance of the monitor. +func New(logTree, mapTree *trillian.Tree, signer *tcrypto.Signer, store *storage.Storage) (*Monitor, error) { + logHasher, err := hashers.NewLogHasher(logTree.GetHashStrategy()) + if err != nil { + return nil, fmt.Errorf("Failed creating LogHasher: %v", err) + } + mapHasher, err := hashers.NewMapHasher(mapTree.GetHashStrategy()) + if err != nil { + return nil, fmt.Errorf("Failed creating MapHasher: %v", err) + } + return &Monitor{ + hasher: mapHasher, + logVerifier: merkle.NewLogVerifier(logHasher), + logPubKey: logTree.GetPublicKey(), + mapPubKey: mapTree.GetPublicKey(), + signer: signer, + store: store, + }, nil +} + +// Process processes a mutation response received from the keytransparency +// server. Processing includes verifying, signing and storing the resulting +// monitoring response. +func (m *Monitor) Process(resp *ktpb.GetMutationsResponse) error { + var smr *trillian.SignedMapRoot + var err error + seen := time.Now().Unix() + errs := m.verifyMutationsResponse(resp) + if len(errs) == 0 { + glog.Infof("Successfully verified mutations response for epoch: %v", resp.Epoch) + smr, err = m.signMapRoot(resp) + if err != nil { + glog.Errorf("Failed to sign map root for epoch %v: %v", resp.Epoch, err) + return fmt.Errorf("m.signMapRoot(_): %v", err) + } + } + if err := m.store.Set(resp.Epoch, seen, smr, resp, errs); err != nil { + glog.Errorf("m.store.Set(%v, %v, _, _, %v): %v", resp.Epoch, seen, errs, err) + return err + } + return nil +} diff --git a/core/monitor/sign.go b/core/monitor/sign.go new file mode 100644 index 000000000..88c32140e --- /dev/null +++ b/core/monitor/sign.go @@ -0,0 +1,37 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitor + +import ( + "fmt" + + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + + "github.com/google/trillian" +) + +func (m *Monitor) signMapRoot(in *ktpb.GetMutationsResponse) (*trillian.SignedMapRoot, error) { + // copy of received SMR: + smr := *in.Smr + smr.Signature = nil + + sig, err := m.signer.SignObject(smr) + if err != nil { + return nil, fmt.Errorf("SignObject(): %v", err) + } + smr.Signature = sig + + return &smr, nil +} diff --git a/core/monitor/storage/storage.go b/core/monitor/storage/storage.go new file mode 100644 index 000000000..dc815d599 --- /dev/null +++ b/core/monitor/storage/storage.go @@ -0,0 +1,96 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "errors" + + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + "github.com/google/trillian" +) + +var ( + // ErrAlreadyStored is raised if the caller tries storing a response which + // has already been stored. + ErrAlreadyStored = errors.New("already stored epoch") + // ErrNotFound is raised if the caller tries to retrieve data for an epoch + // which has not been processed and stored yet. + ErrNotFound = errors.New("data for epoch not found") +) + +// MonitoringResult stores all data +type MonitoringResult struct { + // Smr contains the map root signed by the monitor in case all verifications + // have passed. + Smr *trillian.SignedMapRoot + // Seen is the the unix timestamp at which the mutations response has been + // received. + Seen int64 + // Errors contains a string representation of the verifications steps that + // failed. + Errors []error + // Response contains the original mutations API response from the server + // in case at least one verification step failed. + Response *ktpb.GetMutationsResponse +} + +// Storage is an in-memory store for the monitoring results. +type Storage struct { + store map[int64]*MonitoringResult + latest int64 +} + +// New initializes a +func New() *Storage { + return &Storage{ + store: make(map[int64]*MonitoringResult), + } +} + +// Set internally stores the given data as a MonitoringResult which can be +// retrieved by Get. +func (s *Storage) Set(epoch int64, + seenNanos int64, + smr *trillian.SignedMapRoot, + response *ktpb.GetMutationsResponse, + errorList []error) error { + // see if we already processed this epoch: + if _, ok := s.store[epoch]; ok { + return ErrAlreadyStored + } + // if not we just store the value: + s.store[epoch] = &MonitoringResult{ + Smr: smr, + Seen: seenNanos, + Response: response, + Errors: errorList, + } + s.latest = epoch + return nil +} + +// Get returns the MonitoringResult for the given epoch. It returns an error +// if the result does not exist. +func (s *Storage) Get(epoch int64) (*MonitoringResult, error) { + if result, ok := s.store[epoch]; ok { + return result, nil + } + return nil, ErrNotFound +} + +// LatestEpoch is a convenience method to retrieve the latest stored epoch. +func (s *Storage) LatestEpoch() int64 { + return s.latest +} diff --git a/core/monitor/verify.go b/core/monitor/verify.go new file mode 100644 index 000000000..1deadfb45 --- /dev/null +++ b/core/monitor/verify.go @@ -0,0 +1,31 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package monitor implements the monitor service. A monitor repeatedly polls a +// key-transparency server's Mutations API and signs Map Roots if it could +// reconstruct clients can query. +package monitor + +import ( + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" +) + +// verifyMutationsResponse verifies a response received by the GetMutations API. +// Additionally to the response it takes a complete list of mutations. The list +// of received mutations may differ from those included in the initial response +// because of the max. page size. If any verification check failed it returns +// an error. +func (m *Monitor) verifyMutationsResponse(in *ktpb.GetMutationsResponse) []error { + return nil +} diff --git a/core/monitor/verify_test.go b/core/monitor/verify_test.go new file mode 100644 index 000000000..fb68f4c2d --- /dev/null +++ b/core/monitor/verify_test.go @@ -0,0 +1,18 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitor + +// TODO(ismail): write extensive tests for verification steps, where necessary +// tests should go into integration tests diff --git a/core/mutation/mutation.go b/core/mutation/mutation.go index 2501ccdd7..a8fa2e548 100644 --- a/core/mutation/mutation.go +++ b/core/mutation/mutation.go @@ -212,9 +212,9 @@ func (s *Server) inclusionProofs(ctx context.Context, indexes [][]byte, epoch in glog.Errorf("GetLeaves(): %v", err) return nil, grpc.Errorf(codes.Internal, "Failed fetching map leaf") } - if got, want := len(getResp.MapLeafInclusion), len(indexes); got != want { + if got, want := len(getResp.GetMapLeafInclusion()), len(indexes); got != want { glog.Errorf("GetLeaves() len: %v, want %v", got, want) return nil, grpc.Errorf(codes.Internal, "Failed fetching map leaf") } - return getResp.MapLeafInclusion, nil + return getResp.GetMapLeafInclusion(), nil } diff --git a/core/proto/keytransparency_v1_types/keytransparency_v1_types.pb.go b/core/proto/keytransparency_v1_types/keytransparency_v1_types.pb.go index a220c984b..78a427666 100644 --- a/core/proto/keytransparency_v1_types/keytransparency_v1_types.pb.go +++ b/core/proto/keytransparency_v1_types/keytransparency_v1_types.pb.go @@ -500,7 +500,7 @@ type ListEntryHistoryRequest struct { // app_id is the identifier for the application. AppId string `protobuf:"bytes,4,opt,name=app_id,json=appId" json:"app_id,omitempty"` // first_tree_size is the tree_size of the currently trusted log root. - // Omitting this field will ommit the log consistency proof from the response. + // Omitting this field will omit the log consistency proof from the response. FirstTreeSize int64 `protobuf:"varint,5,opt,name=first_tree_size,json=firstTreeSize" json:"first_tree_size,omitempty"` } @@ -619,7 +619,7 @@ func (m *UpdateEntryRequest) GetEntryUpdate() *EntryUpdate { } // UpdateEntryResponse contains a proof once the update has been included in -// the Merkel Tree. +// the Merkle Tree. type UpdateEntryResponse struct { // proof contains a proof that the update has been included in the tree. Proof *GetEntryResponse `protobuf:"bytes,1,opt,name=proof" json:"proof,omitempty"` diff --git a/core/proto/keytransparency_v1_types/keytransparency_v1_types.proto b/core/proto/keytransparency_v1_types/keytransparency_v1_types.proto index 4b6e603b6..9fb584d58 100644 --- a/core/proto/keytransparency_v1_types/keytransparency_v1_types.proto +++ b/core/proto/keytransparency_v1_types/keytransparency_v1_types.proto @@ -169,7 +169,7 @@ message ListEntryHistoryRequest { // app_id is the identifier for the application. string app_id = 4; // first_tree_size is the tree_size of the currently trusted log root. - // Omitting this field will ommit the log consistency proof from the response. + // Omitting this field will omit the log consistency proof from the response. int64 first_tree_size = 5; } @@ -196,7 +196,7 @@ message UpdateEntryRequest { } // UpdateEntryResponse contains a proof once the update has been included in -// the Merkel Tree. +// the Merkle Tree. message UpdateEntryResponse { // proof contains a proof that the update has been included in the tree. GetEntryResponse proof = 1; @@ -250,4 +250,4 @@ message GetDomainInfoResponse { trillian.Tree map = 2; // Vrf contains the VRF public key. keyspb.PublicKey vrf = 3; -} +} \ No newline at end of file diff --git a/core/proto/monitor_v1_types/gen.go b/core/proto/monitor_v1_types/gen.go new file mode 100644 index 000000000..624949b57 --- /dev/null +++ b/core/proto/monitor_v1_types/gen.go @@ -0,0 +1,17 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate protoc -I=. -I=$GOPATH/src/ -I=$GOPATH/src/github.com/google/trillian/ -I=$GOPATH/src/github.com/googleapis/googleapis --go_out=:. monitor_v1_types.proto + +package monitor_v1_types diff --git a/core/proto/monitor_v1_types/monitor_v1_types.pb.go b/core/proto/monitor_v1_types/monitor_v1_types.pb.go new file mode 100644 index 000000000..79762a88f --- /dev/null +++ b/core/proto/monitor_v1_types/monitor_v1_types.pb.go @@ -0,0 +1,144 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: monitor_v1_types.proto + +/* +Package monitor_v1_types is a generated protocol buffer package. + +Key Transparency Monitor Service + + +It is generated from these files: + monitor_v1_types.proto + +It has these top-level messages: + GetMonitoringRequest + GetMonitoringResponse +*/ +package monitor_v1_types + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import keytransparency_v1_types "github.com/google/keytransparency/core/proto/keytransparency_v1_types" +import trillian "github.com/google/trillian" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// GetMonitoringRequest contains the input parameters of the GetMonitoring APIs. +type GetMonitoringRequest struct { + // epoch specifies the for which the monitoring results will + // be returned (epochs start at 1). + Epoch int64 `protobuf:"varint,1,opt,name=epoch" json:"epoch,omitempty"` + // ktURL is the URL of the keytransparency server for which the monitoring + // result will be returned. + Kt_URL string `protobuf:"bytes,2,opt,name=kt_URL,json=ktURL" json:"kt_URL,omitempty"` +} + +func (m *GetMonitoringRequest) Reset() { *m = GetMonitoringRequest{} } +func (m *GetMonitoringRequest) String() string { return proto.CompactTextString(m) } +func (*GetMonitoringRequest) ProtoMessage() {} +func (*GetMonitoringRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *GetMonitoringRequest) GetEpoch() int64 { + if m != nil { + return m.Epoch + } + return 0 +} + +func (m *GetMonitoringRequest) GetKt_URL() string { + if m != nil { + return m.Kt_URL + } + return "" +} + +type GetMonitoringResponse struct { + // smr contains the map root for the sparse Merkle Tree signed with the + // monitor's key on success. If the checks were not successful the + // smr will be empty. The epochs are encoded into the smr map_revision. + Smr *trillian.SignedMapRoot `protobuf:"bytes,1,opt,name=smr" json:"smr,omitempty"` + // seen_timestamp_nanos contains the time in nanoseconds where this particular + // signed map root was retrieved and processed. The actual timestamp of the + // smr returned by the server is contained in above smr field. + SeenTimestampNanos int64 `protobuf:"varint,2,opt,name=seen_timestamp_nanos,json=seenTimestampNanos" json:"seen_timestamp_nanos,omitempty"` + // errors contains a list of error string representing the verification checks + // that failed while monitoring the key-transparency server. + Errors []string `protobuf:"bytes,3,rep,name=errors" json:"errors,omitempty"` + // error_data contains the original response from the mutations API if and + // only if at least one verification step failed. It can be used to re-run the + // verification steps. + ErrorData *keytransparency_v1_types.GetMutationsResponse `protobuf:"bytes,4,opt,name=error_data,json=errorData" json:"error_data,omitempty"` +} + +func (m *GetMonitoringResponse) Reset() { *m = GetMonitoringResponse{} } +func (m *GetMonitoringResponse) String() string { return proto.CompactTextString(m) } +func (*GetMonitoringResponse) ProtoMessage() {} +func (*GetMonitoringResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *GetMonitoringResponse) GetSmr() *trillian.SignedMapRoot { + if m != nil { + return m.Smr + } + return nil +} + +func (m *GetMonitoringResponse) GetSeenTimestampNanos() int64 { + if m != nil { + return m.SeenTimestampNanos + } + return 0 +} + +func (m *GetMonitoringResponse) GetErrors() []string { + if m != nil { + return m.Errors + } + return nil +} + +func (m *GetMonitoringResponse) GetErrorData() *keytransparency_v1_types.GetMutationsResponse { + if m != nil { + return m.ErrorData + } + return nil +} + +func init() { + proto.RegisterType((*GetMonitoringRequest)(nil), "monitor.v1.types.GetMonitoringRequest") + proto.RegisterType((*GetMonitoringResponse)(nil), "monitor.v1.types.GetMonitoringResponse") +} + +func init() { proto.RegisterFile("monitor_v1_types.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 301 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xc1, 0x6b, 0xc2, 0x30, + 0x18, 0xc5, 0xe9, 0x3a, 0x05, 0x33, 0x18, 0x23, 0xa8, 0x2b, 0x9e, 0xc4, 0x93, 0xbb, 0xa4, 0x73, + 0xfb, 0x13, 0x36, 0xd8, 0x45, 0x77, 0xc8, 0xe6, 0x39, 0xc4, 0xfa, 0x51, 0x83, 0x36, 0x5f, 0x96, + 0x7c, 0x0a, 0xfe, 0xad, 0xfb, 0x67, 0x46, 0x53, 0xed, 0xa1, 0xb0, 0x5b, 0xdf, 0xf7, 0x6b, 0xde, + 0x7b, 0x3c, 0x36, 0xae, 0xd0, 0x1a, 0x42, 0xaf, 0x4e, 0x0b, 0x45, 0x67, 0x07, 0x41, 0x38, 0x8f, + 0x84, 0xfc, 0xe1, 0x72, 0x17, 0xa7, 0x85, 0x88, 0xf7, 0xc9, 0xb6, 0x34, 0xb4, 0x3b, 0x6e, 0x44, + 0x81, 0x55, 0x5e, 0x22, 0x96, 0x07, 0xc8, 0xf7, 0x70, 0x26, 0xaf, 0x6d, 0x70, 0xda, 0x83, 0x2d, + 0xce, 0x79, 0x81, 0x1e, 0xf2, 0xf8, 0xbe, 0x8b, 0x5a, 0xfb, 0x7f, 0x41, 0x93, 0x3b, 0xb9, 0x27, + 0x6f, 0x0e, 0x07, 0xa3, 0x6d, 0xa3, 0x67, 0x6f, 0x6c, 0xf8, 0x01, 0xb4, 0x6a, 0xca, 0x18, 0x5b, + 0x4a, 0xf8, 0x39, 0x42, 0x20, 0x3e, 0x64, 0x3d, 0x70, 0x58, 0xec, 0xb2, 0x64, 0x9a, 0xcc, 0x53, + 0xd9, 0x08, 0x3e, 0x62, 0xfd, 0x3d, 0xa9, 0xb5, 0x5c, 0x66, 0x37, 0xd3, 0x64, 0x3e, 0x90, 0xbd, + 0x3d, 0xad, 0xe5, 0x72, 0xf6, 0x9b, 0xb0, 0x51, 0xc7, 0x25, 0x38, 0xb4, 0x01, 0xf8, 0x13, 0x4b, + 0x43, 0xe5, 0xa3, 0xc9, 0xdd, 0xcb, 0xa3, 0x68, 0xc3, 0xbf, 0x4c, 0x69, 0x61, 0xbb, 0xd2, 0x4e, + 0x22, 0x92, 0xac, 0xff, 0xe1, 0xcf, 0x6c, 0x18, 0x00, 0xac, 0x22, 0x53, 0x41, 0x20, 0x5d, 0x39, + 0x65, 0xb5, 0xc5, 0x10, 0x93, 0x52, 0xc9, 0x6b, 0xf6, 0x7d, 0x45, 0x9f, 0x35, 0xe1, 0x63, 0xd6, + 0x07, 0xef, 0xd1, 0x87, 0x2c, 0x9d, 0xa6, 0xf3, 0x81, 0xbc, 0x28, 0xbe, 0x62, 0x2c, 0x7e, 0xa9, + 0xad, 0x26, 0x9d, 0xdd, 0xc6, 0x6c, 0x21, 0x3a, 0xc3, 0xb4, 0xc3, 0x8b, 0xba, 0xf9, 0x91, 0x34, + 0x19, 0xb4, 0xe1, 0x5a, 0x5c, 0x0e, 0xa2, 0xc3, 0xbb, 0x26, 0xbd, 0xe9, 0xc7, 0xa5, 0x5e, 0xff, + 0x02, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xdc, 0x29, 0x97, 0xcb, 0x01, 0x00, 0x00, +} diff --git a/core/proto/monitor_v1_types/monitor_v1_types.proto b/core/proto/monitor_v1_types/monitor_v1_types.proto new file mode 100644 index 000000000..48a964601 --- /dev/null +++ b/core/proto/monitor_v1_types/monitor_v1_types.proto @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +// Key Transparency Monitor Service +// +package monitor.v1.types; + +import "github.com/google/keytransparency/core/proto/keytransparency_v1_types/keytransparency_v1_types.proto"; +import "trillian.proto"; + +// GetMonitoringRequest contains the input parameters of the GetMonitoring APIs. +message GetMonitoringRequest { + // epoch specifies the for which the monitoring results will + // be returned (epochs start at 1). + int64 epoch = 1; + + // ktURL is the URL of the keytransparency server for which the monitoring + // result will be returned. + string kt_URL = 2; +} + +message GetMonitoringResponse { + // smr contains the map root for the sparse Merkle Tree signed with the + // monitor's key on success. If the checks were not successful the + // smr will be empty. The epochs are encoded into the smr map_revision. + trillian.SignedMapRoot smr = 1; + + // seen_timestamp_nanos contains the time in nanoseconds where this particular + // signed map root was retrieved and processed. The actual timestamp of the + // smr returned by the server is contained in above smr field. + int64 seen_timestamp_nanos = 2; + + // + // The following fields provide more information about each failure in this + // response, if any. + // + + // errors contains a list of error string representing the verification checks + // that failed while monitoring the key-transparency server. + repeated string errors = 3; + + // error_data contains the original response from the mutations API if and + // only if at least one verification step failed. It can be used to re-run the + // verification steps. + keytransparency.v1.types.GetMutationsResponse error_data = 4; + } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 21e23b369..7298bd312 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,3 +149,26 @@ services: - --max-period=5m - --alsologtostderr - --v=5 + + kt-monitor: + depends_on: + - kt-server + - kt-signer + build: + context: .. + dockerfile: ./keytransparency/cmd/keytransparency-monitor/Dockerfile + entrypoint: + - /go/bin/keytransparency-monitor + - --addr=0.0.0.0:8099 + - --kt-url=kt-server:8080 + - --poll-period=5s + - --tls-key=genfiles/server.key + - --tls-cert=genfiles/server.crt + - --sign-key=/kt/monitor_sign-key.pem + - --password=towel + - --alsologtostderr + - --v=3 + image: us.gcr.io/key-transparency/keytransparency-monitor + restart: always + ports: + - "8099:8099" # gRPC / HTTPS diff --git a/impl/monitor/client/client.go b/impl/monitor/client/client.go new file mode 100644 index 000000000..18245eb76 --- /dev/null +++ b/impl/monitor/client/client.go @@ -0,0 +1,105 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +// +// This file contains Monitor'c grpc client logic: poll mutations from the +// kt-server mutations API and page if necessary. +// + +import ( + "time" + + "github.com/golang/glog" + "golang.org/x/net/context" + "google.golang.org/grpc" + + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + mupb "github.com/google/keytransparency/impl/proto/mutation_v1_service" +) + +// Each page contains pageSize profiles. Each profile contains multiple +// keys. Assuming 2 keys per profile (each of size 2048-bit), a page of +// size 16 will contain about 8KB of data. +const pageSize = 16 + +// Client queries the server side mutations API. +type Client struct { + client mupb.MutationServiceClient + pollPeriod time.Duration +} + +// New initializes a new mutations API monitoring client. +func New(client mupb.MutationServiceClient, pollPeriod time.Duration) *Client { + return &Client{ + client: client, + pollPeriod: pollPeriod, + } +} + +// StartPolling initiates polling the server side mutations API. It does not +// block returns a channel. +// The caller should listen on the channel to receiving the latest polled +// mutations response including all paged mutations. If anything went wrong +// while polling the response channel contains an error. +func (c *Client) StartPolling(startEpoch int64) (<-chan *ktpb.GetMutationsResponse, <-chan error) { + response := make(chan *ktpb.GetMutationsResponse) + errChan := make(chan error) + go func() { + epoch := startEpoch + t := time.NewTicker(c.pollPeriod) + for now := range t.C { + glog.Infof("Polling: %v", now) + // time out if we exceed the poll period: + ctx, _ := context.WithTimeout(context.Background(), c.pollPeriod) + monitorResp, err := c.pollMutations(ctx, epoch) + if err != nil { + glog.Infof("pollMutations(_): %v", err) + errChan <- err + } else { + // only write to results channel and increment epoch + // if there was no error: + glog.Infof("Got response %v at %v", monitorResp.Epoch, now) + response <- monitorResp + epoch++ + } + } + }() + return response, errChan +} + +func (c *Client) pollMutations(ctx context.Context, + queryEpoch int64, + opts ...grpc.CallOption) (*ktpb.GetMutationsResponse, error) { + response, err := c.client.GetMutations(ctx, &ktpb.GetMutationsRequest{ + PageSize: pageSize, + Epoch: queryEpoch, + }, opts...) + if err != nil { + return nil, err + } + + // Page if necessary: query all mutations in the current epoch + for response.GetNextPageToken() != "" { + req := &ktpb.GetMutationsRequest{PageSize: pageSize} + resp, err := c.client.GetMutations(ctx, req, opts...) + if err != nil { + return nil, err + } + response.Mutations = append(response.Mutations, resp.GetMutations()...) + } + + return response, nil +} diff --git a/impl/monitor/monitor_test.go b/impl/monitor/monitor_test.go new file mode 100644 index 000000000..20aab42e2 --- /dev/null +++ b/impl/monitor/monitor_test.go @@ -0,0 +1,35 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package monitor implements the monitor service. A monitor repeatedly polls a +// key-transparency server's Mutations API and signs Map Roots if it could +// reconstruct +// clients can query. +package monitor + +import ( + "testing" + + "golang.org/x/net/context" + + "github.com/google/keytransparency/core/monitor/storage" +) + +func TestGetSignedMapRoot(t *testing.T) { + srv := New(storage.New()) + _, err := srv.GetSignedMapRoot(context.TODO(), nil) + if got, want := err, ErrNothingProcessed; got != want { + t.Errorf("GetSignedMapRoot(_, _): %v, want %v", got, want) + } +} diff --git a/impl/monitor/server.go b/impl/monitor/server.go new file mode 100644 index 000000000..c2f25b551 --- /dev/null +++ b/impl/monitor/server.go @@ -0,0 +1,102 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package monitor implements the monitor service. A monitor repeatedly polls a +// key-transparency server's Mutations API and signs Map Roots if it could +// reconstruct +// clients can query. + +// Package monitor contains an implementation of a Monitor server which can be +// queried for monitoring results. +package monitor + +import ( + "errors" + + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + + "github.com/google/keytransparency/core/monitor/storage" + mopb "github.com/google/keytransparency/core/proto/monitor_v1_types" +) + +var ( + // ErrNothingProcessed occurs when the monitor did not process any mutations / + // smrs yet. + ErrNothingProcessed = errors.New("did not process any mutations yet") +) + +// Server holds internal state for the monitor server. It serves monitoring +// responses via a grpc and HTTP API. +type Server struct { + storage *storage.Storage +} + +// New creates a new instance of the monitor server. +func New(storage *storage.Storage) *Server { + return &Server{ + storage: storage, + } +} + +// GetSignedMapRoot returns the latest valid signed map root the monitor +// observed. Additionally, the response contains additional data necessary to +// reproduce errors on failure. +// +// Returns the signed map root for the latest epoch the monitor observed. If +// the monitor could not reconstruct the map root given the set of mutations +// from the previous to the current epoch it won't sign the map root and +// additional data will be provided to reproduce the failure. +func (s *Server) GetSignedMapRoot(ctx context.Context, in *mopb.GetMonitoringRequest) (*mopb.GetMonitoringResponse, error) { + latestEpoch := s.storage.LatestEpoch() + if latestEpoch == 0 { + return nil, ErrNothingProcessed + } + return s.getResponseByRevision(latestEpoch) +} + +// GetSignedMapRootByRevision works similar to GetSignedMapRoot but returns +// the monitor's result for a specific map revision. +// +// Returns the signed map root for the specified epoch the monitor observed. +// If the monitor could not reconstruct the map root given the set of +// mutations from the previous to the current epoch it won't sign the map root +// and additional data will be provided to reproduce the failure. +func (s *Server) GetSignedMapRootByRevision(ctx context.Context, in *mopb.GetMonitoringRequest) (*mopb.GetMonitoringResponse, error) { + return s.getResponseByRevision(in.GetEpoch()) +} + +func (s *Server) getResponseByRevision(epoch int64) (*mopb.GetMonitoringResponse, error) { + res, err := s.storage.Get(epoch) + if err == storage.ErrNotFound { + return nil, grpc.Errorf(codes.NotFound, + "Could not find monitoring response for epoch %d", epoch) + } + + resp := &mopb.GetMonitoringResponse{ + Smr: res.Smr, + SeenTimestampNanos: res.Seen, + } + + if len(res.Errors) > 0 { + for _, err := range res.Errors { + resp.Errors = append(resp.Errors, err.Error()) + } + // data to replay the verification steps: + resp.ErrorData = res.Response + } + + return resp, nil +} diff --git a/impl/mutation/mutation.go b/impl/mutation/mutation.go index 9fae43072..8b816ef12 100644 --- a/impl/mutation/mutation.go +++ b/impl/mutation/mutation.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package mutation implements the monitor service. +// Package mutation implements the mutations service a monitor can query. package mutation import ( diff --git a/impl/proto/monitor_v1_service/gen.go b/impl/proto/monitor_v1_service/gen.go new file mode 100644 index 000000000..250a32e56 --- /dev/null +++ b/impl/proto/monitor_v1_service/gen.go @@ -0,0 +1,19 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate protoc -I=. -I=$GOPATH/src/ -I=$GOPATH/src/github.com/google/trillian/ -I=$GOPATH/src/github.com/googleapis/googleapis/ --go_out=,plugins=grpc:. monitor_v1_service.proto + +//go:generate protoc -I=. -I=$GOPATH/src/ -I=$GOPATH/src/github.com/google/trillian/ -I=$GOPATH/src/github.com/googleapis/googleapis/ --grpc-gateway_out=logtostderr=true:. monitor_v1_service.proto + +package monitor_v1_service diff --git a/impl/proto/monitor_v1_service/monitor_v1_service.pb.go b/impl/proto/monitor_v1_service/monitor_v1_service.pb.go new file mode 100644 index 000000000..1f6e270b0 --- /dev/null +++ b/impl/proto/monitor_v1_service/monitor_v1_service.pb.go @@ -0,0 +1,198 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: monitor_v1_service.proto + +/* +Package monitor_v1_service is a generated protocol buffer package. + +Monitor Service + +The Key Transparency monitor server service consists of APIs to fetch +monitor results queried using the mutations API. + +It is generated from these files: + monitor_v1_service.proto + +It has these top-level messages: +*/ +package monitor_v1_service + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import monitor_v1_types "github.com/google/keytransparency/core/proto/monitor_v1_types" +import _ "google.golang.org/genproto/googleapis/api/annotations" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for MonitorService service + +type MonitorServiceClient interface { + // GetSignedMapRoot returns the latest valid signed map root the monitor + // observed. Additionally, the response contains additional data necessary to + // reproduce errors on failure. + // + // Returns the signed map root for the latest epoch the monitor observed. If + // the monitor could not reconstruct the map root given the set of mutations + // from the previous to the current epoch it won't sign the map root and + // additional data will be provided to reproduce the failure. + GetSignedMapRoot(ctx context.Context, in *monitor_v1_types.GetMonitoringRequest, opts ...grpc.CallOption) (*monitor_v1_types.GetMonitoringResponse, error) + // GetSignedMapRootByRevision works similar to GetSignedMapRoot but returns + // the monitor's result for a specific map revision. + // + // Returns the signed map root for the specified epoch the monitor observed. + // If the monitor could not reconstruct the map root given the set of + // mutations from the previous to the current epoch it won't sign the map root + // and additional data will be provided to reproduce the failure. + GetSignedMapRootByRevision(ctx context.Context, in *monitor_v1_types.GetMonitoringRequest, opts ...grpc.CallOption) (*monitor_v1_types.GetMonitoringResponse, error) +} + +type monitorServiceClient struct { + cc *grpc.ClientConn +} + +func NewMonitorServiceClient(cc *grpc.ClientConn) MonitorServiceClient { + return &monitorServiceClient{cc} +} + +func (c *monitorServiceClient) GetSignedMapRoot(ctx context.Context, in *monitor_v1_types.GetMonitoringRequest, opts ...grpc.CallOption) (*monitor_v1_types.GetMonitoringResponse, error) { + out := new(monitor_v1_types.GetMonitoringResponse) + err := grpc.Invoke(ctx, "/monitor.v1.service.MonitorService/GetSignedMapRoot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *monitorServiceClient) GetSignedMapRootByRevision(ctx context.Context, in *monitor_v1_types.GetMonitoringRequest, opts ...grpc.CallOption) (*monitor_v1_types.GetMonitoringResponse, error) { + out := new(monitor_v1_types.GetMonitoringResponse) + err := grpc.Invoke(ctx, "/monitor.v1.service.MonitorService/GetSignedMapRootByRevision", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for MonitorService service + +type MonitorServiceServer interface { + // GetSignedMapRoot returns the latest valid signed map root the monitor + // observed. Additionally, the response contains additional data necessary to + // reproduce errors on failure. + // + // Returns the signed map root for the latest epoch the monitor observed. If + // the monitor could not reconstruct the map root given the set of mutations + // from the previous to the current epoch it won't sign the map root and + // additional data will be provided to reproduce the failure. + GetSignedMapRoot(context.Context, *monitor_v1_types.GetMonitoringRequest) (*monitor_v1_types.GetMonitoringResponse, error) + // GetSignedMapRootByRevision works similar to GetSignedMapRoot but returns + // the monitor's result for a specific map revision. + // + // Returns the signed map root for the specified epoch the monitor observed. + // If the monitor could not reconstruct the map root given the set of + // mutations from the previous to the current epoch it won't sign the map root + // and additional data will be provided to reproduce the failure. + GetSignedMapRootByRevision(context.Context, *monitor_v1_types.GetMonitoringRequest) (*monitor_v1_types.GetMonitoringResponse, error) +} + +func RegisterMonitorServiceServer(s *grpc.Server, srv MonitorServiceServer) { + s.RegisterService(&_MonitorService_serviceDesc, srv) +} + +func _MonitorService_GetSignedMapRoot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(monitor_v1_types.GetMonitoringRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MonitorServiceServer).GetSignedMapRoot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/monitor.v1.service.MonitorService/GetSignedMapRoot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MonitorServiceServer).GetSignedMapRoot(ctx, req.(*monitor_v1_types.GetMonitoringRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MonitorService_GetSignedMapRootByRevision_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(monitor_v1_types.GetMonitoringRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MonitorServiceServer).GetSignedMapRootByRevision(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/monitor.v1.service.MonitorService/GetSignedMapRootByRevision", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MonitorServiceServer).GetSignedMapRootByRevision(ctx, req.(*monitor_v1_types.GetMonitoringRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _MonitorService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "monitor.v1.service.MonitorService", + HandlerType: (*MonitorServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetSignedMapRoot", + Handler: _MonitorService_GetSignedMapRoot_Handler, + }, + { + MethodName: "GetSignedMapRootByRevision", + Handler: _MonitorService_GetSignedMapRootByRevision_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "monitor_v1_service.proto", +} + +func init() { proto.RegisterFile("monitor_v1_service.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 273 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x90, 0xb1, 0x4e, 0xc3, 0x40, + 0x0c, 0x86, 0xd5, 0x0e, 0x0c, 0x19, 0x10, 0xba, 0x09, 0x45, 0x4c, 0x0c, 0x14, 0x18, 0x62, 0x0a, + 0x1b, 0x23, 0x4b, 0x17, 0xba, 0xa4, 0x30, 0x47, 0xd7, 0x60, 0xa5, 0xa7, 0x26, 0xf6, 0x71, 0x76, + 0x22, 0x45, 0x55, 0x17, 0x5e, 0x01, 0x76, 0x9e, 0x87, 0x99, 0x57, 0xe0, 0x41, 0x90, 0x92, 0x20, + 0x55, 0x30, 0x74, 0x61, 0xf5, 0xef, 0xff, 0xf3, 0x27, 0x47, 0xc7, 0x15, 0x93, 0x53, 0x0e, 0x59, + 0x33, 0xcd, 0x04, 0x43, 0xe3, 0x72, 0x4c, 0x7c, 0x60, 0x65, 0x63, 0x86, 0x24, 0x69, 0xa6, 0xc9, + 0x90, 0xc4, 0x0f, 0x85, 0xd3, 0x55, 0xbd, 0x4c, 0x72, 0xae, 0xa0, 0x60, 0x2e, 0x4a, 0x84, 0x35, + 0xb6, 0x1a, 0x2c, 0x89, 0xb7, 0x01, 0x29, 0x6f, 0x21, 0xe7, 0x80, 0xd0, 0x11, 0x60, 0x07, 0xad, + 0xad, 0x47, 0xf9, 0x33, 0xe8, 0x2f, 0xc5, 0x27, 0x03, 0xca, 0x7a, 0x07, 0x96, 0x88, 0xd5, 0xaa, + 0x63, 0x1a, 0xd2, 0xeb, 0x8f, 0x71, 0x74, 0x38, 0xef, 0x8b, 0x8b, 0x5e, 0xc3, 0xbc, 0x8d, 0xa2, + 0xa3, 0x19, 0xea, 0xc2, 0x15, 0x84, 0x4f, 0x73, 0xeb, 0x53, 0x66, 0x35, 0x67, 0xc9, 0x8e, 0x70, + 0x8f, 0x9f, 0xa1, 0x0e, 0x4d, 0x47, 0x45, 0x8a, 0xcf, 0x35, 0x8a, 0xc6, 0x93, 0xbd, 0x7b, 0xe2, + 0x99, 0x04, 0x4f, 0xe1, 0xe5, 0xf3, 0xeb, 0x75, 0x7c, 0x61, 0x26, 0xd0, 0x4c, 0x7f, 0xd4, 0x61, + 0xb3, 0xd6, 0xec, 0x31, 0xbd, 0xdf, 0x42, 0x65, 0x3d, 0x04, 0x94, 0xba, 0x54, 0xb9, 0x2d, 0xad, + 0xa2, 0xa8, 0x79, 0x1f, 0x45, 0xf1, 0x6f, 0xad, 0xbb, 0x36, 0xc5, 0xc6, 0x89, 0x63, 0xfa, 0x7f, + 0xc1, 0xab, 0x4e, 0xf0, 0xd2, 0x9c, 0xef, 0x13, 0x84, 0x0d, 0x7a, 0xce, 0x57, 0xdb, 0xe5, 0x41, + 0xf7, 0xd2, 0x9b, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07, 0x96, 0x6d, 0xf3, 0xf6, 0x01, 0x00, + 0x00, +} diff --git a/impl/proto/monitor_v1_service/monitor_v1_service.pb.gw.go b/impl/proto/monitor_v1_service/monitor_v1_service.pb.gw.go new file mode 100644 index 000000000..69269299b --- /dev/null +++ b/impl/proto/monitor_v1_service/monitor_v1_service.pb.gw.go @@ -0,0 +1,203 @@ +// Code generated by protoc-gen-grpc-gateway +// source: monitor_v1_service.proto +// DO NOT EDIT! + +/* +Package monitor_v1_service is a reverse proxy. + +It translates gRPC into RESTful JSON APIs. +*/ +package monitor_v1_service + +import ( + "io" + "net/http" + + "github.com/golang/protobuf/proto" + "github.com/google/keytransparency/core/proto/monitor_v1_types" + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "github.com/grpc-ecosystem/grpc-gateway/utilities" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" +) + +var _ codes.Code +var _ io.Reader +var _ = runtime.String +var _ = utilities.NewDoubleArray + +var ( + filter_MonitorService_GetSignedMapRoot_0 = &utilities.DoubleArray{Encoding: map[string]int{"kt_URL": 0}, Base: []int{1, 1, 0}, Check: []int{0, 1, 2}} +) + +func request_MonitorService_GetSignedMapRoot_0(ctx context.Context, marshaler runtime.Marshaler, client MonitorServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq monitor_v1_types.GetMonitoringRequest + var metadata runtime.ServerMetadata + + var ( + val string + ok bool + err error + _ = err + ) + + val, ok = pathParams["kt_URL"] + if !ok { + return nil, metadata, grpc.Errorf(codes.InvalidArgument, "missing parameter %s", "kt_URL") + } + + protoReq.Kt_URL, err = runtime.String(val) + + if err != nil { + return nil, metadata, err + } + + if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_MonitorService_GetSignedMapRoot_0); err != nil { + return nil, metadata, grpc.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.GetSignedMapRoot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func request_MonitorService_GetSignedMapRootByRevision_0(ctx context.Context, marshaler runtime.Marshaler, client MonitorServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq monitor_v1_types.GetMonitoringRequest + var metadata runtime.ServerMetadata + + var ( + val string + ok bool + err error + _ = err + ) + + val, ok = pathParams["kt_URL"] + if !ok { + return nil, metadata, grpc.Errorf(codes.InvalidArgument, "missing parameter %s", "kt_URL") + } + + protoReq.Kt_URL, err = runtime.String(val) + + if err != nil { + return nil, metadata, err + } + + val, ok = pathParams["epoch"] + if !ok { + return nil, metadata, grpc.Errorf(codes.InvalidArgument, "missing parameter %s", "epoch") + } + + protoReq.Epoch, err = runtime.Int64(val) + + if err != nil { + return nil, metadata, err + } + + msg, err := client.GetSignedMapRootByRevision(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +// RegisterMonitorServiceHandlerFromEndpoint is same as RegisterMonitorServiceHandler but +// automatically dials to "endpoint" and closes the connection when "ctx" gets done. +func RegisterMonitorServiceHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { + conn, err := grpc.Dial(endpoint, opts...) + if err != nil { + return err + } + defer func() { + if err != nil { + if cerr := conn.Close(); cerr != nil { + grpclog.Printf("Failed to close conn to %s: %v", endpoint, cerr) + } + return + } + go func() { + <-ctx.Done() + if cerr := conn.Close(); cerr != nil { + grpclog.Printf("Failed to close conn to %s: %v", endpoint, cerr) + } + }() + }() + + return RegisterMonitorServiceHandler(ctx, mux, conn) +} + +// RegisterMonitorServiceHandler registers the http handlers for service MonitorService to "mux". +// The handlers forward requests to the grpc endpoint over "conn". +func RegisterMonitorServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + client := NewMonitorServiceClient(conn) + + mux.Handle("GET", pattern_MonitorService_GetSignedMapRoot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, req) + if err != nil { + runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + } + resp, md, err := request_MonitorService_GetSignedMapRoot_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + return + } + + forward_MonitorService_GetSignedMapRoot_0(ctx, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + mux.Handle("GET", pattern_MonitorService_GetSignedMapRootByRevision_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, req) + if err != nil { + runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + } + resp, md, err := request_MonitorService_GetSignedMapRootByRevision_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + return + } + + forward_MonitorService_GetSignedMapRootByRevision_0(ctx, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + return nil +} + +var ( + pattern_MonitorService_GetSignedMapRoot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3, 2, 4}, []string{"v1", "monitor", "kt_URL", "map", "results"}, "latest")) + + pattern_MonitorService_GetSignedMapRootByRevision_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3, 2, 4, 1, 0, 4, 1, 5, 5}, []string{"v1", "monitor", "kt_URL", "map", "results", "epoch"}, "")) +) + +var ( + forward_MonitorService_GetSignedMapRoot_0 = runtime.ForwardResponseMessage + + forward_MonitorService_GetSignedMapRootByRevision_0 = runtime.ForwardResponseMessage +) diff --git a/impl/proto/monitor_v1_service/monitor_v1_service.proto b/impl/proto/monitor_v1_service/monitor_v1_service.proto new file mode 100644 index 000000000..eae54884a --- /dev/null +++ b/impl/proto/monitor_v1_service/monitor_v1_service.proto @@ -0,0 +1,61 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +// Monitor Service +// +// The Key Transparency monitor server service consists of APIs to fetch +// monitor results queried using the mutations API. +package monitor.v1.service; + +import "github.com/google/keytransparency/core/proto/monitor_v1_types/monitor_v1_types.proto"; +import "google/api/annotations.proto"; + +// The Monitor Service API allows clients to query the monitors observed and +// validated signed map roots. +// +// - Signed Map Roots can be collected using the GetSignedMapRoot APIs. +// - Monitor resources are named: +// - /v1/monitor/{kt-url}/map/results/{epoch} +// - /v1/monitor/{kt-url}/map/results:latest +// +service MonitorService { + // GetSignedMapRoot returns the latest valid signed map root the monitor + // observed. Additionally, the response contains additional data necessary to + // reproduce errors on failure. + // + // Returns the signed map root for the latest epoch the monitor observed. If + // the monitor could not reconstruct the map root given the set of mutations + // from the previous to the current epoch it won't sign the map root and + // additional data will be provided to reproduce the failure. + rpc GetSignedMapRoot(monitor.v1.types.GetMonitoringRequest) + returns (monitor.v1.types.GetMonitoringResponse) { + option (google.api.http) = { get: "/v1/monitor/{kt_URL}/map/results:latest" }; + } + + // GetSignedMapRootByRevision works similar to GetSignedMapRoot but returns + // the monitor's result for a specific map revision. + // + // Returns the signed map root for the specified epoch the monitor observed. + // If the monitor could not reconstruct the map root given the set of + // mutations from the previous to the current epoch it won't sign the map root + // and additional data will be provided to reproduce the failure. + rpc GetSignedMapRootByRevision(monitor.v1.types.GetMonitoringRequest) + returns(monitor.v1.types.GetMonitoringResponse) { + option (google.api.http) = { get: "/v1/monitor/{kt_URL}/map/results/{epoch}" }; + } +} + + diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d16dc471b..95b8dfda7 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -73,7 +73,7 @@ function pushTrillianImgs() function pushKTImgs() { - images=("keytransparency-server" "prometheus") + images=("keytransparency-server" "keytransparency-monitor" "prometheus") for DOCKER_IMAGE_NAME in "${images[@]}" do # Push the images as we refer to them in the kubernetes config files: diff --git a/scripts/gen_monitor_keys.sh b/scripts/gen_monitor_keys.sh new file mode 100755 index 000000000..fad8f0be8 --- /dev/null +++ b/scripts/gen_monitor_keys.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create output directory. +mkdir -p "${GOPATH}/src/github.com/google/keytransparency/genfiles" +cd "${GOPATH}/src/github.com/google/keytransparency/genfiles" + +INTERACTIVE=1 + +while getopts ":f" opt; do + case $opt in + f) + INTERACTIVE=0 + ;; + \?) + echo "Invalid option: -$OPTARG." + usage + exit 1 + ;; + esac +done + + +DEFAULT_PWD=towel + +# Generate monitor signing key-pair: +if ((INTERACTIVE == 1)); then + # Prompts for password: + openssl ecparam -name prime256v1 -genkey | openssl ec -aes256 -out monitor_sign-key.pem +else + openssl ecparam -name prime256v1 -genkey | openssl ec -aes256 -passout pass:$DEFAULT_PWD -out monitor_sign-key.pem +fi +chmod 600 monitor_sign-key.pem + diff --git a/scripts/prepare_server.sh b/scripts/prepare_server.sh index d7a2a745f..581d61077 100755 --- a/scripts/prepare_server.sh +++ b/scripts/prepare_server.sh @@ -23,6 +23,7 @@ INTERACTIVE=1 FRONTEND=1 FRONTENDNUM=0 BACKEND=1 +MONITOR=1 # 1 means SQLite, 2 means MySQL DSN="" IP1="127.0.0.1" @@ -131,6 +132,10 @@ if ((FRONTEND == 1)); then ./scripts/gen_server_keys.sh -d "${CERTDOMAIN}" -a "${CERTIP}" -s "${SAN_DNS}" fi +if ((MONITOR == 1)); then + ./scripts/gen_monitor_keys.sh -f +fi + # Generating .env file ENV="SIGN_PERIOD=5" diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write.go b/vendor/github.com/pelletier/go-toml/tomltree_write.go index 6a7fa1745..4df87eba0 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_write.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_write.go @@ -4,11 +4,11 @@ import ( "bytes" "fmt" "io" + "reflect" "sort" "strconv" "strings" "time" - "reflect" ) // encodes a string to a TOML-compliant string value diff --git a/vendor/github.com/spf13/viper/viper.go b/vendor/github.com/spf13/viper/viper.go index 31b41a6b2..ee1e6c6b7 100644 --- a/vendor/github.com/spf13/viper/viper.go +++ b/vendor/github.com/spf13/viper/viper.go @@ -53,7 +53,7 @@ func init() { type remoteConfigFactory interface { Get(rp RemoteProvider) (io.Reader, error) Watch(rp RemoteProvider) (io.Reader, error) - WatchChannel(rp RemoteProvider)(<-chan *RemoteResponse, chan bool) + WatchChannel(rp RemoteProvider) (<-chan *RemoteResponse, chan bool) } // RemoteConfig is optional, see the remote package