Examples: Fix the interactive test for MacOS users (#4779)

* Bump e2e version Signed-off-by: Matej Gera <[email protected]> * Update docs Signed-off-by: Matej Gera <[email protected]> * Update CHANGELOG Signed-off-by: Matej Gera <[email protected]> * Bump e2e version; fix cp -t usage Signed-off-by: Matej Gera <[email protected]> * Introduce variable block plan profile Signed-off-by: Matej Gera <[email protected]> * Update example docs Signed-off-by: Matej Gera <[email protected]> * Re-comment the skip line Signed-off-by: Matej Gera <[email protected]>
thanos-io · Nov 3, 2021 · 1c3b984 · 1c3b984
1 parent f61bcf5
commit 1c3b984
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4663](https://github.com/thanos-io/thanos/pull/4663) Fetcher: Fix discovered data races.
 - [#4754](https://github.com/thanos-io/thanos/pull/4754) Query: Fix possible panic on stores endpoint.
 - [#4753](https://github.com/thanos-io/thanos/pull/4753) Store: validate block sync concurrency parameter
+- [#4779](https://github.com/thanos-io/thanos/pull/4779) Examples: Fix the interactive test for MacOS users.
 - [#4792](https://github.com/thanos-io/thanos/pull/4792) Store: Fix data race in BucketedBytes pool.
 - [#4769](https://github.com/thanos-io/thanos/pull/4769) Query-frontend+api: add "X-Request-ID" field and other fields to start call log.
 

diff --git a/examples/interactive/interactive_test.go b/examples/interactive/interactive_test.go
@@ -23,7 +23,10 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-const data = "data"
+const (
+	data           = "data"
+	defaultProfile = "continuous-30d-tiny"
+)
 
 var (
 	maxTimeFresh = `2021-07-27T00:00:00Z`
@@ -42,7 +45,17 @@ func exec(cmd string, args ...string) error {
 	return nil
 }
 
+// createData generates some blocks for us to play with and makes them
+// available to store and Prometheus instances.
+//
+// You can choose different profiles by setting the BLOCK_PROFILE environment variable.
+// Available profiles can be found at https://github.com/thanos-io/thanosbench/blob/master/pkg/blockgen/profiles.go#L28
 func createData() (perr error) {
+	profile := os.Getenv("BLOCK_PROFILE")
+	if profile == "" {
+		profile = defaultProfile
+	}
+
 	fmt.Println("Re-creating data (can take minutes)...")
 	defer func() {
 		if perr != nil {
@@ -53,33 +66,33 @@ func createData() (perr error) {
 	if err := exec(
 		"sh", "-c",
 		fmt.Sprintf("mkdir -p %s && "+
-			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p continuous-1w-small --labels 'cluster=\"eu-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
-			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", store1Data, maxTimeOld, store1Data),
+			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p %s --labels 'cluster=\"eu-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
+			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", store1Data, profile, maxTimeOld, store1Data),
 	); err != nil {
 		return err
 	}
 	if err := exec(
 		"sh", "-c",
 		fmt.Sprintf("mkdir -p %s && "+
-			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p continuous-1w-small --labels 'cluster=\"us-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
-			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", store2Data, maxTimeOld, store2Data),
+			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p %s --labels 'cluster=\"us-1\"' --labels 'replica=\"0\"' --max-time=%s | "+
+			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", store2Data, profile, maxTimeOld, store2Data),
 	); err != nil {
 		return err
 	}
 
 	if err := exec(
 		"sh", "-c",
 		fmt.Sprintf("mkdir -p %s && "+
-			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p continuous-1w-small --max-time=%s | "+
-			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", prom1Data, maxTimeFresh, prom1Data),
+			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p %s --max-time=%s | "+
+			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", prom1Data, profile, maxTimeFresh, prom1Data),
 	); err != nil {
 		return err
 	}
 	if err := exec(
 		"sh", "-c",
 		fmt.Sprintf("mkdir -p %s && "+
-			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p continuous-1w-small --max-time=%s | "+
-			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", prom2Data, maxTimeFresh, prom2Data),
+			"docker run -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block plan -p %s --max-time=%s | "+
+			"docker run -v %s/:/shared -i quay.io/thanos/thanosbench:v0.2.0-rc.1 block gen --output.dir /shared", prom2Data, profile, maxTimeFresh, prom2Data),
 	); err != nil {
 		return err
 	}
@@ -91,12 +104,13 @@ func createData() (perr error) {
 func TestReadOnlyThanosSetup(t *testing.T) {
 	t.Skip("This is interactive test - it will run until you will kill it or curl 'finish' endpoint. Comment and run as normal test to use it!")
 
-	// Create 20k series for 2w of TSDB blocks. Cache them to 'data' dir so we don't need to re-create on every run (it takes ~5m).
+	// Create series of TSDB blocks. Cache them to 'data' dir so we don't need to re-create on every run.
 	_, err := os.Stat(data)
 	if os.IsNotExist(err) {
 		testutil.Ok(t, createData())
 	} else {
 		testutil.Ok(t, err)
+		fmt.Println("Skipping blocks generation, found data directory.")
 	}
 
 	e, err := e2e.NewDockerEnvironment("interactive")
@@ -225,7 +239,7 @@ func TestReadOnlyThanosSetup(t *testing.T) {
 	sidecar2 := e2edb.NewThanosSidecar(e, "sidecar2", prom2, e2edb.WithImage("thanos:latest"))
 
 	testutil.Ok(t, exec("cp", "-r", prom1Data+"/.", promHA0.Dir()))
-	testutil.Ok(t, exec("sh", "-c", "find "+prom1Data+"/ -maxdepth 1 -type d | tail -5 | xargs cp -r -t "+promHA1.Dir())) // Copy only 5 blocks from 9 to mimic replica 1 with partial data set.
+	testutil.Ok(t, exec("sh", "-c", "find "+prom1Data+"/ -maxdepth 1 -type d | tail -5 | xargs -I {} cp -r {} "+promHA1.Dir())) // Copy only 5 blocks from 9 to mimic replica 1 with partial data set.
 	testutil.Ok(t, exec("cp", "-r", prom2Data+"/.", prom2.Dir()))
 
 	testutil.Ok(t, promHA0.SetConfig(`
@@ -310,7 +324,6 @@ global:
 	)
 	testutil.Ok(t, e2e.StartAndWaitReady(query1))
 
-	// Let's start the party with 1.5 billions of samples (~20k series for 15s scrape for 2w).
 	// Wait until we have 5 gRPC connections.
 	testutil.Ok(t, query1.WaitSumMetricsWithOptions(e2e.Equals(5), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics()))
 

diff --git a/go.mod b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/chromedp/chromedp v0.5.3
 	github.com/cortexproject/cortex v1.10.1-0.20211006150606-fb15b432e267
 	github.com/davecgh/go-spew v1.1.1
-	github.com/efficientgo/e2e v0.11.1-0.20210829161758-f4cc6dbdc6ea
+	github.com/efficientgo/e2e v0.11.2-0.20211027134903-67d538984a47
 	github.com/efficientgo/tools/core v0.0.0-20210129205121-421d0828c9a6
 	github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb

diff --git a/go.sum b/go.sum
@@ -529,8 +529,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/efficientgo/e2e v0.11.1-0.20210829161758-f4cc6dbdc6ea h1:PFKVWZOnEfthNTcdpiz0wGzmikPOINnbRYEo4MneloE=
-github.com/efficientgo/e2e v0.11.1-0.20210829161758-f4cc6dbdc6ea/go.mod h1:vDnF4AAEZmO0mvyFIATeDJPFaSRM7ywaOnKd61zaSoE=
+github.com/efficientgo/e2e v0.11.2-0.20211027134903-67d538984a47 h1:k0qDUhOU0KJqKztQYJL1qMBR9nCOntuIRWYwA56Z634=
+github.com/efficientgo/e2e v0.11.2-0.20211027134903-67d538984a47/go.mod h1:vDnF4AAEZmO0mvyFIATeDJPFaSRM7ywaOnKd61zaSoE=
 github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a h1:Az9zRvQubUIHE+tHAm0gG7Dwge08V8Q/9uNSIFjFm+A=
 github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M=
 github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 h1:kM/ALyvAnTrwSB+nlKqoKaDnZbInp1YImZvW+gtHwc8=

diff --git a/tutorials/interactive-example/README.md b/tutorials/interactive-example/README.md
@@ -1,8 +1,16 @@
 # Interactive Example
 
-The interactive example in `examples/interactive` can be run by Linux users. It is not yet supported for MacOS users. Refer to [#4642](https://github.com/thanos-io/thanos/issues/4642) to track the status of this issue.   
+Thanos repository contains an interactive example, which will spin up an environment for you locally in Docker containers.
+
+A prerequiste for running the example is having [Docker](https://docs.docker.com/get-docker/) installed already.
 
 To run the example, run the following commands:
-1. Navigate to the `examples/interactive` directory.
-2. Comment [this line](https://github.com/thanos-io/thanos/blob/bd134d7a823708fa135e7a6931e76f581be5f879/examples/interactive/interactive_test.go#L92) in the file `interactive_test.go`.   
-3. `go test interactive_test.go -test.timeout=9999m`.  
+1. Build the Thanos image locally by running `make docker`.
+2. Navigate to the `examples/interactive` directory.
+3. Comment [this line](https://github.com/thanos-io/thanos/blob/bd134d7a823708fa135e7a6931e76f581be5f879/examples/interactive/interactive_test.go#L92) in the file `interactive_test.go`.
+4. `go test interactive_test.go -test.timeout=9999m`.
+
+The example will generate some data for you to play with and store it in `data` directory for subsequent tests runs.
+You can choose from different hardcoded generation profiles, which will give you different amount of data. You can change this by setting the `BLOCK_PROFILE` environment variable to a selected profile name. You can find the available profiles [here](https://github.com/thanos-io/thanosbench/blob/master/pkg/blockgen/profiles.go#L28) (we use `thanosbench` tool to generate our test data).
+
+The default profile is `continuous-30d-tiny` which will give you test data for 5 different applications with single metric. If you really want to ramp it up, you can use `continuous-30d-tiny`, which will give you test data for 100 applications with 100 metrics for each (BEWARE: Generating this much data requires a lot of RAM, make sure you have at least 8 GB availble, otherwise generation might fail).