Merge pull request #1 from NBISweden/dev/efficient

Make log dumping smarter

elasticsearch/.gitignore

@@ -0,0 +1,3 @@
+data
+datasecond
+main

elasticsearch/README.md

@@ -1,36 +1,45 @@
-## Elasticsearch backups
+# Elasticsearch backups
+
+## Create a key
+Enable the transit encryption engine and create a key. Give it a descriptive name.
+
+## Create some indices in ES
+```sh
+./main --action create --index index123
+```
 
 ## Dumping encrypted index to S3
 ```sh
-./main dump
+./main --action dump --index index123-mon-jan-8-17-43-24
 s3cmd ls -c s3conf s3://dumps
 s3cmd get -c s3conf s3://dumps/indexname
 ```
 
-## Loading index from S3
+## Loading index from S3 to ES
 ```sh
-./main load
+./main --action load --index index123-mon-jan-8-17-43-24
 ```
 
-## Script configuration
+## Example script configuration
 ```yaml
 s3:
   url: "https://localhost"
   port: 9000
   accesskey: "myaccesskey"
-  secretkey: "mysecretkey!0"
+  secretkey: "mysecretkey"
   bucket: "dumps"
-  cacert: "./certs/s3.pem"
+  #chunksize: 32
+  cacert: "./certs/ca.pem"
 elastic:
-  #addr: "http://localhost:9200"
-  addr: "http://localhost:9201"
+  ## INSTANCE 1
+  addr: "http://localhost:9200"
+  ## INSTANCE 2
+  #addr: "http://localhost:9201"
   user: "elastic"
   password: "elastic"
-  #index: "my-test-index"
-  index: "my-test-index-mon-jan-6-11-26-58"
 vault:
   addr: "http://localhost:8282"
-  token: "s.lepNY18TQPM1QRZR9kW1DAhS"
+  token: ""
   transitpath: "transit"
   key: "transit"
 ```

elasticsearch/config.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"flag"
 	"path"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 )
 
@@ -26,6 +28,21 @@ func NewConfig() *Config {
 	return c
 }
 
+// getCLflags returns the CL args of indexName and action
+func getCLflags() (string, string) {
+	flag.String("action", "create", "action can be create, dump or load")
+	flag.String("index", "index123", "index name to create, dump or load")
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+	viper.BindPFlags(pflag.CommandLine)
+
+	indexName := viper.GetString("index")
+	action := viper.GetString("action")
+	return indexName, action
+
+}
+
 // configS3Storage populates a S3Config
 func configS3Storage() S3Config {
 	s3 := S3Config{}
@@ -72,7 +89,6 @@ func configElastic() ElasticConfig {
 	elastic.Addr = viper.GetString("elastic.addr")
 	elastic.User = viper.GetString("elastic.user")
 	elastic.Password = viper.GetString("elastic.password")
-	elastic.Index = viper.GetString("elastic.index")
 
 	return elastic
 }

elasticsearch/config.yaml

@@ -2,19 +2,19 @@ s3:
   url: "https://localhost"
   port: 9000
   accesskey: "myaccesskey"
-  secretkey: "mysecretkey!0"
+  secretkey: "mysecretkey"
   bucket: "dumps"
   #chunksize: 32
-  cacert: "./certs/s3.pem"
+  cacert: "./certs/ca.pem"
 elastic:
-  #addr: "http://localhost:9200"
-  addr: "http://localhost:9201"
+  ## INSTANCE 1
+  addr: "http://localhost:9200"
+  ## INSTANCE 2
+  #addr: "http://localhost:9201"
   user: "elastic"
   password: "elastic"
-  #index: "my-test-index"
-  index: "my-test-index-mon-jan-6-11-26-58"
 vault:
   addr: "http://localhost:8282"
-  token: "s.lepNY18TQPM1QRZR9kW1DAhS"
+  token: "s.7rQlPx4SI1Ke82wZBEPXeDLu"
   transitpath: "transit"
   key: "transit"

elasticsearch/crypto.go

@@ -1,11 +1,17 @@
 package main
 
 import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
 	"fmt"
+	"io"
+	"strings"
 
 	vault "github.com/mittwald/vaultgo"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/guregu/null.v3"
+	"github.com/tidwall/gjson"
 )
 
 // VaultConfig holds Vault settings
@@ -16,21 +22,10 @@ type VaultConfig struct {
 	Key              string
 }
 
-func encryptIndex(c *vault.Client, index string, mountpath string, key string) string {
-	const rsa4096 = "rsa-4096"
-
-	fmt.Println(c.Token())
+func getKey(c *vault.Client, mountpath string, key string) string {
 
 	transit := c.TransitWithMountPoint(mountpath)
 
-	err := transit.Create(key, &vault.TransitCreateOptions{
-		Exportable: null.BoolFrom(true),
-		Type:       rsa4096,
-	})
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	res, err := transit.Read(key)
 	if err != nil {
 		log.Fatal(err)
@@ -44,31 +39,65 @@ func encryptIndex(c *vault.Client, index string, mountpath string, key string) s
 	if err != nil {
 		log.Fatal(err)
 	}
-	log.Printf("%v+", exportRes.Data.Keys[1])
-
-	encryptResponse, err := transit.Encrypt(key, &vault.TransitEncryptOptions{
-		Plaintext: index,
-	})
 
+	decodedKey, err := base64.StdEncoding.DecodeString(exportRes.Data.Keys[1])
 	if err != nil {
-		log.Fatalf("Error occurred during encryption: %v", err)
+		log.Fatalf("Error occurred during decoding: %v", err)
 	}
 
-	return encryptResponse.Data.Ciphertext
+	return string(decodedKey)
 }
 
-func descryptIndex(c *vault.Client, encIndex string, mountpath string, key string) string {
+func encryptDocs(hits gjson.Result, stream cipher.Stream, fr io.Writer) {
+	var res strings.Builder
+	fmt.Fprintf(&res, "%s\n", hits.Raw)
+	plainText := []byte(res.String())
+	cipherText := make([]byte, len(plainText))
+	stream.XORKeyStream(cipherText, plainText)
 
-	const rsa4096 = "rsa-4096"
+	if _, err := io.Copy(fr, bytes.NewReader(cipherText)); err != nil {
+		log.Fatal(err)
+	}
 
-	transit := c.TransitWithMountPoint(mountpath)
+}
 
-	decryptResponse, err := transit.Decrypt(key, &vault.TransitDecryptOptions{
-		Ciphertext: encIndex,
-	})
+func decryptDocs(rc io.ReadCloser, key []byte) string {
+	buf := new(bytes.Buffer)
+	_, err := buf.ReadFrom(rc)
+	data := buf.Bytes()
 	if err != nil {
-		log.Fatalf("Error occurred during decryption: %v", err)
+		log.Error(err)
 	}
 
-	return decryptResponse.Data.Plaintext
+	stream := getStreamDecryptor(key)
+
+	// XORKeyStream can work in-place if the two arguments are the same.
+	stream.XORKeyStream(data, data)
+
+	out := string(data)
+	return out
+}
+
+func getStreamEncryptor(key []byte) cipher.Stream {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var iv [aes.BlockSize]byte
+	if err != nil {
+		log.Fatal(err)
+	}
+	stream := cipher.NewCFBEncrypter(block, iv[:])
+
+	return stream
+}
+
+func getStreamDecryptor(key []byte) cipher.Stream {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var iv [aes.BlockSize]byte
+	stream := cipher.NewCFBDecrypter(block, iv[:])
+	return stream
 }

elasticsearch/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     container_name: mys3
     environment:
       - MINIO_ACCESS_KEY=myaccesskey
-      - MINIO_SECRET_KEY=mysecretkey!0
+      - MINIO_SECRET_KEY=mysecretkey
     healthcheck:
       test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"]
       interval: 5s