pb.proto

/*
 * Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


// Style guide for Protocol Buffer 3.
// Use CamelCase (with an initial capital) for message names – for example,
// SongServerRequest. Use underscore_separated_names for field names – for
// example, song_name.

syntax = "proto3";

package pb;

import "api.proto";
import "github.com/dgraph-io/badger/v3/pb/pb.proto";
import "github.com/gogo/protobuf/gogoproto/gogo.proto";

/* import "gogoproto/gogo.proto"; */

/* option (gogoproto.marshaler_all) = true; */
/* option (gogoproto.sizer_all) = true; */
/* option (gogoproto.unmarshaler_all) = true; */
/* option (gogoproto.goproto_getters_all) = true; */

message List {
	repeated fixed64 uids = 1;
}

message TaskValue {
	bytes val = 1;
	Posting.ValType val_type = 2;
}

message SrcFunction {
	string name = 1;
	repeated string args = 3;
	bool isCount = 4;
}

message Query {
	string attr = 1;
	repeated string langs = 2; // language list for attribute
	fixed64 after_uid = 3;  // Only return UIDs greater than this.
	bool do_count = 4;      // Are we just getting lengths?

	// Exactly one of uids and terms is populated.
	List uid_list = 5;

	// Function to generate or filter UIDs.
	SrcFunction src_func = 6;

	bool reverse = 7;  // Whether this is a reverse edge.

	FacetParams facet_param = 8; // which facets to fetch
	FilterTree facets_filter = 9; // filtering on facets : has Op (and/or/not) tree

	bool expand_all = 10; // expand all language variants.

	uint64 read_ts = 13;
	int32 cache = 14;
	int32 first = 15; // used to limit the number of result. Typically, the count is value of first
	// field. Now, It's been used only for has query.
}

message ValueList {
	repeated TaskValue values = 1;
}

message LangList {
	repeated string lang = 1;
}

message Result {
	repeated List uid_matrix = 1;
	repeated ValueList value_matrix = 2;
	repeated uint32 counts = 3;
	bool intersect_dest = 4;
	repeated FacetsList facet_matrix = 5;
	repeated LangList lang_matrix = 6;
	bool list = 7;
}

message Order {
	string attr = 1;
	bool desc = 2;
	repeated string langs = 3;
}

message SortMessage {
	repeated Order order = 1;
	repeated List uid_matrix = 2;
	int32 count = 3;   // Return this many elements.
	int32 offset = 4;  // Skip this many elements.

	uint64 read_ts = 13;
}

message SortResult {
	repeated List uid_matrix = 1;
}

message RaftContext {
	fixed64 id = 1;
	uint32 group = 2;
	string addr = 3;
	uint64 snapshot_ts = 4;
	bool is_learner = 5;
}

// Member stores information about RAFT group member for a single RAFT node.
// Note that each server can be serving multiple RAFT groups. Each group would have
// one RAFT node per server serving that group.
message Member {
	fixed64 id = 1;
	uint32 group_id = 2 [(gogoproto.jsontag) = "groupId,omitempty"];
	string addr = 3;
	bool leader = 4;
	bool am_dead = 5 [(gogoproto.jsontag) = "amDead,omitempty"];
	uint64 last_update = 6 [(gogoproto.jsontag) = "lastUpdate,omitempty"];
	bool learner = 7;

	bool cluster_info_only = 13 [(gogoproto.jsontag) = "clusterInfoOnly,omitempty"];
	bool force_group_id = 14 [(gogoproto.jsontag) = "forceGroupId,omitempty"];
}

message Group {
	map<uint64, Member> members = 1; // Raft ID is the key.
	map<string, Tablet> tablets = 2; // Predicate + others are key.
	uint64 snapshot_ts          = 3; // Stores Snapshot transaction ts.
	uint64 checksum             = 4; // Stores a checksum.
	uint64 checkpoint_ts        = 5; // Stores checkpoint ts as seen by leader.
}

message License {
	string user = 1;
	uint64 maxNodes = 2;
	int64 expiryTs = 3;
	bool enabled = 4;
}

message ZeroProposal {
	reserved 8;  // Was used for string key.
	map<uint32, uint64> snapshot_ts = 1; // Group ID -> Snapshot Ts.
	Member member = 2;
	Tablet tablet = 3;
	uint64 maxUID = 4;
	uint64 maxTxnTs = 5;
	uint64 maxNsID = 12;
	uint64 maxRaftId = 6;
	api.TxnContext txn = 7;
	string cid = 9; // Used as unique identifier for the cluster.
	License license = 10;
	ZeroSnapshot snapshot = 11; // Used to make Zeros take a snapshot.
	// 12 has already been used.
}

// MembershipState is used to pack together the current membership state of all the nodes
// in the caller server; and the membership updates recorded by the callee server since
// the provided lastUpdate.
message MembershipState {
	uint64 counter = 1;  // used to find latest membershipState in case of race.
	map<uint32, Group> groups = 2;
	map<uint64, Member> zeros = 3;
	uint64 maxUID = 4;
	uint64 maxTxnTs = 5;
	uint64 maxNsID = 10;
	uint64 maxRaftId = 6;
	repeated Member removed = 7;
	string cid = 8; // Used to uniquely identify the Dgraph cluster.
	License license = 9;
	// 10 has already been used.
}

message ConnectionState {
    Member member = 1;
    MembershipState state = 2;
    uint64 max_pending = 3; // Used to determine the timstamp for reading after bulk load
}

message HealthInfo {
    string instance = 1;
    string address = 2;
    string status = 3;
    string group = 4; // string so group = 0 can be printed in JSON.
    string version = 5;
    int64 uptime = 6;
    int64 lastEcho = 7;
    repeated string ongoing = 8;
    repeated string indexing = 9;
    repeated string ee_features = 10;
		uint64 max_assigned = 11;
}

message Tablet {
    uint32 group_id = 1 [(gogoproto.jsontag) = "groupId,omitempty"]; // Served by which group.
    string predicate = 2;
    bool force = 3; // Used while moving predicate.
    int64 on_disk_bytes = 7;
    bool remove = 8;
    bool read_only = 9 [(gogoproto.jsontag) = "readOnly,omitempty"]; // If true, do not ask zero to serve any tablets.
	uint64 move_ts = 10 [(gogoproto.jsontag) = "moveTs,omitempty"];
	int64 uncompressed_bytes = 11; // Estimated uncompressed size of tablet in bytes
}

message DirectedEdge {
	fixed64 entity             = 1;    // Subject or source node / UID.
	string attr                = 2;       // Attribute or predicate. Labels the edge.
	bytes value                = 3;       // Edge points to a value.
	Posting.ValType value_type = 4;  // The type of the value
	fixed64 value_id           = 5;   // Object or destination node / UID.
	string label               = 6;
	string lang                = 7;
	enum Op {
		SET = 0;
		DEL = 1;
	}
	Op op = 8;
	repeated api.Facet facets = 9;
	repeated string allowedPreds = 10;
}

message Mutations {
	uint32 group_id	= 1;
	uint64 start_ts	= 2;
	repeated DirectedEdge edges	= 3;
	repeated SchemaUpdate schema = 4;
	repeated TypeUpdate types	= 6;
	enum DropOp {
		NONE = 0;
		ALL = 1;
		DATA = 2;
		TYPE = 3;
	}
	DropOp drop_op = 7;
	string drop_value = 8;

	Metadata metadata = 9;
}

message Metadata {
  // HintType represents a hint that will be passed along the mutation and used
  // to add the predicate to the schema if it's not already there.
  enum HintType {
    // DEFAULT means no hint is provided and Dgraph will follow the default behavior.
    DEFAULT = 0;
    // SINGLE signals that the predicate should be created as a single type (e.g string, uid).
    SINGLE = 1;
    // LIST signals that the predicate should be created as a list (e.g [string], [uid]).
    LIST = 2;
  }

  // Map of predicates to their hints.
	map<string, HintType> pred_hints = 1;
}

message Snapshot {
	RaftContext context = 1;
	uint64 index = 2;
	uint64 read_ts = 3;
	// done is used to indicate that snapshot stream was a success.
	bool done	= 4;
	// since_ts stores the ts of the last snapshot to support diff snap updates.
	uint64 since_ts = 5;
}

message ZeroSnapshot {
	uint64 index = 1;
	uint64 checkpoint_ts = 2;
	MembershipState state = 5;
}

message RestoreRequest {
	uint32 group_id = 1;
	uint64 restore_ts = 2;
	string location = 3;
	string backup_id = 4;

	// Credentials when using a minio or S3 bucket as the backup location.
	string access_key = 5;
	string secret_key = 6;
	string session_token = 7;
	bool anonymous = 8;

	// Info needed to process encrypted backups.
	string encryption_key_file = 9;
	// Vault options
	string vault_addr = 10;
	string vault_roleid_file = 11;
	string vault_secretid_file = 12;
	string vault_path = 13;
	string vault_field = 14;
	string vault_format = 15;

	uint64 backup_num = 16;
}

message Proposal {
	reserved 7; // Was used for string key.
	Mutations mutations    		= 2;
	repeated badgerpb3.KV kv  = 4;
	MembershipState state  		= 5;
	string clean_predicate 		= 6;  // Delete the predicate which was moved to other group.
	OracleDelta delta      		= 8;
	Snapshot snapshot      		= 9;  // Used to tell the group when to take snapshot.
	uint64 index           		= 10; // Used to store Raft index, in raft.Ready.
	uint64 expected_checksum 	= 11; // Block an operation until membership reaches this checksum.
	RestoreRequest restore 		= 12;
	uint64 CDCMinReadTs 			= 13;
}

message KVS {
	bytes data = 5;

	// done used to indicate if the stream of KVS is over.
	bool done	= 2;
	// predicates is the list of predicates known by the leader at the time of the snapshot.
	repeated string predicates = 3;
	// types is the list of types known by the leader at the time of the snapshot.
	repeated string types = 4;
}

// Posting messages.
message Posting {
	fixed64 uid = 1;
	bytes value = 2;
	enum ValType {
		DEFAULT = 0;
		BINARY = 1;
		INT = 2; // We treat it as int64.
		FLOAT = 3;
		BOOL = 4;
		DATETIME = 5;
		GEO = 6;
		UID = 7;
		PASSWORD = 8;
		STRING = 9;
    OBJECT = 10;
	}
	ValType val_type = 3;
	enum PostingType {
		REF=0;          // UID
		VALUE=1;        // simple, plain value
		VALUE_LANG=2;   // value with specified language
	}
	PostingType posting_type = 4;
	bytes lang_tag = 5; // Only set for VALUE_LANG
	string label = 6;
	repeated api.Facet facets = 9;

	// TODO: op is only used temporarily. See if we can remove it from here.
	uint32 op = 12;
	uint64 start_ts = 13;   // Meant to use only inmemory
	uint64 commit_ts = 14;  // Meant to use only inmemory
}

message UidBlock {
	uint64 base = 1;
	// deltas contains the deltas encoded with Varints. We don't store deltas as a list of integers,
	// because when the PB is brought to memory, Go would always use 8-bytes per integer. Instead,
	// storing it as a byte slice is a lot cheaper in memory.
	bytes deltas = 2;
	// num_uids is the number of UIDs in the block. We are including this because we want to
	// switch encoding to groupvarint encoding. Current avaialble open source version implements
	// encoding and decoding for uint32. To use that, we create different blocks for different 32-bit
	// MSB base uids. That is, if the 32 MSBs are different, we will create a new block irrespective
	// of whether the block is filled with the block_size or not.
	// Default Blocksize is 256 so uint32 would be sufficient.
	uint32 num_uids = 3;
}

message UidPack {
	uint32 block_size        = 1;
	repeated UidBlock blocks = 2;
	uint64 alloc_ref = 23; // This field should be set to 0 during marshal.
}

message PostingList {
	UidPack pack = 1; // Encoded list of uids in this posting list.
	repeated Posting postings = 2;
	uint64 commit_ts = 3; // More inclination towards smaller values.

  repeated uint64 splits = 4;
}

message FacetParam {
	string key = 1;
	string alias = 2;
}

message FacetParams {
	bool all_keys = 1; // keys should be in sorted order.
	repeated FacetParam param = 2;
}

message Facets {
	repeated api.Facet facets = 1;
}

message FacetsList {
	repeated Facets facets_list = 1;
}

message Function {
	string name = 1;          // Name of the function : eq, le
	string key = 2;           // Facet key over which to run the function.
	repeated string args = 3; // Arguments of the function.
}

// Op and Children are internal nodes and Func on leaves.
message FilterTree {
	string op = 1;
	repeated FilterTree children = 2;
	Function func = 3;
}

// Schema messages.
message SchemaRequest {
	uint32 group_id = 1;
	repeated string predicates = 2;
	// fields can be on of type, index, reverse or tokenizer
	repeated string fields = 3;

  repeated string types = 4;
}

message SchemaNode {
	string predicate = 1;
	string type = 2;
	bool index = 3;
	repeated string tokenizer = 4;
	bool reverse = 5;
	bool count = 6;
	bool list = 7;
	bool upsert = 8;
	bool lang = 9;
	bool no_conflict = 10;
}

message SchemaResult {
	repeated SchemaNode schema = 1 [deprecated=true];
}

message SchemaUpdate {
	string predicate = 1;
	Posting.ValType value_type = 2;
	enum Directive {
	   NONE = 0;
	   INDEX = 1;
	   REVERSE = 2;
	   DELETE = 3;
	}
	Directive directive = 3;
	repeated string tokenizer = 4;
	bool count = 5;
	bool list = 6;
	bool upsert = 8;
	bool lang = 9;

	// Fields required for type system.
	bool non_nullable = 10;
	bool non_nullable_list = 11;

	// If value_type is OBJECT, then this represents an object type with a
	// custom name. This field stores said name.
	string object_type_name = 12;

	bool no_conflict = 13;

	// Deleted field:
	reserved 7;
	reserved "explicit";
}

message TypeUpdate {
	string type_name = 1;
	repeated SchemaUpdate fields = 2;
}

message MapHeader {
	repeated bytes partition_keys = 1;
}

message MovePredicatePayload {
	string predicate         = 1;
	uint32 source_gid        = 2;
	uint32 dest_gid          = 3;
	uint64 txn_ts            = 4;
	uint64 expected_checksum = 5;
}

message TxnStatus {
	uint64 start_ts = 1;
	uint64 commit_ts = 2;
}

message OracleDelta {
	repeated TxnStatus txns             = 1;
	uint64 max_assigned                 = 2;
	map<uint32, uint64> group_checksums = 3;
	// implement tmax.
}

message TxnTimestamps {
	repeated uint64 ts = 1;
}

message PeerResponse {
	bool status = 1;
}

message RaftBatch {
	RaftContext context = 1;
	api.Payload payload = 2;
}

service Raft {
	rpc Heartbeat (api.Payload)        returns (stream HealthInfo) {}
	rpc RaftMessage (stream RaftBatch) returns (api.Payload) {}
	rpc JoinCluster (RaftContext)      returns (api.Payload) {}
	rpc IsPeer (RaftContext)           returns (PeerResponse) {}
}

service Zero {
	// These 3 endpoints are for handling membership.
	rpc Connect (Member)               returns (ConnectionState) {}
	rpc UpdateMembership (Group)                 returns (api.Payload) {}
	rpc StreamMembership (api.Payload)       returns (stream MembershipState) {}

	rpc Oracle (api.Payload)           returns (stream OracleDelta) {}
	rpc ShouldServe (Tablet)           returns (Tablet) {}
	rpc AssignIds (Num)               returns (AssignedIds) {}
	rpc Timestamps (Num)               returns (AssignedIds) {}
	rpc CommitOrAbort (api.TxnContext) returns (api.TxnContext) {}
	rpc TryAbort (TxnTimestamps)       returns (OracleDelta) {}
}

service Worker {
	// Data serving RPCs.
	rpc Mutate (Mutations)                  returns (api.TxnContext) {}
	rpc ServeTask (Query)                   returns (Result) {}
	rpc StreamSnapshot (stream Snapshot)    returns (stream KVS) {}
	rpc Sort (SortMessage)                  returns (SortResult) {}
	rpc Schema (SchemaRequest)              returns (SchemaResult) {}
	rpc Backup (BackupRequest)              returns (BackupResponse) {}
	rpc Restore (RestoreRequest)            returns (Status) {}
	rpc Export (ExportRequest)              returns (ExportResponse) {}
	rpc ReceivePredicate(stream KVS)        returns (api.Payload) {}
	rpc MovePredicate(MovePredicatePayload) returns (api.Payload) {}
	rpc Subscribe(SubscriptionRequest) returns (stream badgerpb3.KVList) {}
	rpc UpdateGraphQLSchema(UpdateGraphQLSchemaRequest) returns (UpdateGraphQLSchemaResponse) {}
}

message SubscriptionRequest {
	repeated bytes prefixes = 1;
}

message SubscriptionResponse {
    badgerpb3.KVList kvs = 1;
}

message Num {
	uint64 val = 1;
	bool read_only = 2;
	bool forwarded = 3; // True if this request was forwarded by a peer.
	enum leaseType {
		NS_ID = 0;
		UID = 1;
		TXN_TS = 2;
	}
	leaseType type = 4;
}

message AssignedIds {
	uint64 startId = 1;
	uint64 endId = 2;

	// The following is used for read only transactions.
	uint64 read_only = 5;
}

message SnapshotMeta {
	uint64 client_ts = 1;
	uint32 group_id = 2;
}

// Status describes a general status response.
// code: 0 = success, 0 != failure.
message Status {
	int32 code = 1;
	string msg = 2;
}

message BackupRequest {
	uint64 read_ts = 1;
  uint64 since_ts = 2;
	uint32 group_id = 3;
	string unix_ts = 4;
	string destination = 5;
	string access_key = 6;
	string secret_key = 7;
	string session_token = 8;

	// True if no credentials should be used to access the S3 or minio bucket.
	// For example, when using a bucket with a public policy.
	bool anonymous = 9;

	// The predicates to backup. All other predicates present in the group (e.g
	// stale data from a predicate move) will be ignored.
	repeated string predicates = 10;
}

message BackupResponse {
	repeated DropOperation drop_operations = 1;
}

message DropOperation {
	enum DropOp {
		ALL = 0;
		DATA = 1;
		ATTR = 2;
	}
	DropOp drop_op    = 1;
	// When drop_op is ATTR, drop_value will be the name of the ATTR; empty otherwise.
	string drop_value = 2;
}

message ExportRequest {
	uint32  group_id = 1;  // Group id to back up.
	uint64  read_ts  = 2;
	int64   unix_ts  = 3;
	string  format   = 4;

	string destination = 5;

	// These credentials are used to access the S3 or minio bucket.
	string access_key = 6;
	string secret_key = 7;
	string session_token = 8;
	bool anonymous = 9;
}

message ExportResponse {
	// 0 indicates a success, and a non-zero code indicates failure
	int32 code = 1;
	string msg = 2;
	repeated string files = 3;
}

// A key stored in the format used for writing backups.
message BackupKey {
  enum KeyType {
    UNKNOWN = 0;
    DATA = 1;
    INDEX = 2;
    REVERSE = 3;
    COUNT = 4;
    COUNT_REV = 5;
    SCHEMA = 6;
    TYPE = 7;
  }

  KeyType type = 1;
  string attr = 2;
  uint64 uid = 3;
  uint64 start_uid = 4;
  string term = 5;
  uint32 count = 6;
}

// A posting list stored in the format used for writing backups.
message BackupPostingList {
	repeated uint64 uids = 1;
	repeated Posting postings = 2;
	uint64 commit_ts = 3;
  repeated uint64 splits = 4;
	bytes uid_bytes = 5;
}

message UpdateGraphQLSchemaRequest {
	uint64                start_ts       = 1;
	string                graphql_schema = 2;
	repeated SchemaUpdate dgraph_preds   = 3;
	repeated TypeUpdate   dgraph_types   = 4;
}

message UpdateGraphQLSchemaResponse {
	uint64 uid = 1;
}

// BulkMeta stores metadata from the map phase of the bulk loader.
message BulkMeta {
	int64 edge_count = 1;
	map<string, SchemaUpdate> schema_map = 2;
}

// vim: noexpandtab sw=2 ts=2