Parquet Store gateway (#7046)

* parquet store gateway poc

Signed-off-by: yeya24 <benye@amazon.com>

* refactor and fix test

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* fix test

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* delete unused code

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* add spans

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* rollback InitialSync

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* Rollback wait stability max/min duration

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* Add inflight request limit not checked test

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* integate parquet bucket storage test with existing test

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* fix lint

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* fix lint

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* add changelog

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* fix changelog

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* rollback bucket store metrics

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* delete unused fields

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* fix lint

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

* move changelog

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

---------

Signed-off-by: yeya24 <benye@amazon.com>
Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>
Co-authored-by: yeya24 <benye@amazon.com>

Author: SungJin1212

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] StoreGateway: Introduces a new parquet mode. #7046
 * [ENHANCEMENT] Ingester: Add `enable_matcher_optimization` config to apply low selectivity matchers lazily. #7063
 * [ENHANCEMENT] Distributor: Add a label references validation for remote write v2 request. #7074
 * [ENHANCEMENT] Distributor: Add count, spans, and buckets validations for native histogram. #7072

docs/blocks-storage/querier.md

@@ -1729,6 +1729,10 @@ blocks_storage:
     # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy
     [block_discovery_strategy: <string> | default = "concurrent"]
 
+    # Type of bucket store to use (tsdb or parquet).
+    # CLI flag: -blocks-storage.bucket-store.bucket-store-type
+    [bucket_store_type: <string> | default = "tsdb"]
+
     # Max size - in bytes - of a chunks pool, used to reduce memory allocations.
     # The pool is shared across all tenants. 0 to disable the limit.
     # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes

docs/blocks-storage/store-gateway.md

@@ -1815,6 +1815,10 @@ blocks_storage:
     # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy
     [block_discovery_strategy: <string> | default = "concurrent"]
 
+    # Type of bucket store to use (tsdb or parquet).
+    # CLI flag: -blocks-storage.bucket-store.bucket-store-type
+    [bucket_store_type: <string> | default = "tsdb"]
+
     # Max size - in bytes - of a chunks pool, used to reduce memory allocations.
     # The pool is shared across all tenants. 0 to disable the limit.
     # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes

docs/configuration/config-file-reference.md

@@ -2401,6 +2401,10 @@ bucket_store:
   # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy
   [block_discovery_strategy: <string> | default = "concurrent"]
 
+  # Type of bucket store to use (tsdb or parquet).
+  # CLI flag: -blocks-storage.bucket-store.bucket-store-type
+  [bucket_store_type: <string> | default = "tsdb"]
+
   # Max size - in bytes - of a chunks pool, used to reduce memory allocations.
   # The pool is shared across all tenants. 0 to disable the limit.
   # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/minio/minio-go/v7 v7.0.93
 	github.com/mitchellh/go-wordwrap v1.0.1
-	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/oklog/ulid v1.3.1
 	github.com/opentracing-contrib/go-grpc v0.1.2
 	github.com/opentracing-contrib/go-stdlib v1.1.0
 	github.com/opentracing/opentracing-go v1.2.0

integration/e2ecortex/services.go

@@ -204,6 +204,41 @@ func NewIngesterWithConfigFile(name string, store RingStore, address, configFile
 	)
 }
 
+func NewParquetConverter(name string, store RingStore, address string, flags map[string]string, image string) *CortexService {
+	return NewParquetConverterWithConfigFile(name, store, address, "", flags, image)
+}
+
+func NewParquetConverterWithConfigFile(name string, store RingStore, address, configFile string, flags map[string]string, image string) *CortexService {
+	if configFile != "" {
+		flags["-config.file"] = filepath.Join(e2e.ContainerSharedDir, configFile)
+	}
+
+	// Configure the ingesters ring backend
+	flags["-ring.store"] = string(store)
+	switch store {
+	case RingStoreConsul:
+		flags["-consul.hostname"] = address
+	case RingStoreEtcd:
+		flags["-etcd.endpoints"] = address
+	}
+
+	if image == "" {
+		image = GetDefaultImage()
+	}
+
+	return NewCortexService(
+		name,
+		image,
+		e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{
+			"-target":    "parquet-converter",
+			"-log.level": "warn",
+		}, flags))...),
+		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
+		httpPort,
+		grpcPort,
+	)
+}
+
 func NewQueryFrontend(name string, flags map[string]string, image string) *CortexService {
 	return NewQueryFrontendWithConfigFile(name, "", flags, image)
 }