This is an automated email from the ASF dual-hosted git repository. adheipsingh pushed a commit to branch 1.3.0-debug in repository https://gitbox.apache.org/repos/asf/druid-operator.git
commit 36892e40d48c459609def50a6f5ee3501ee0ed3b Author: AdheipSingh <[email protected]> AuthorDate: Fri Sep 26 18:22:44 2025 +0530 update debug logs --- README.md | 12 +- apis/druid/v1alpha1/druid_types.go | 16 -- apis/druid/v1alpha1/zz_generated.deepcopy.go | 10 - chart/Chart.yaml | 4 +- chart/crds/druid.apache.org_druidingestions.yaml | 8 - chart/crds/druid.apache.org_druids.yaml | 88 ------- .../bases/druid.apache.org_druidingestions.yaml | 8 - config/crd/bases/druid.apache.org_druids.yaml | 88 ------- controllers/druid/druid_controller.go | 6 +- controllers/druid/handler.go | 2 - controllers/druid/handler_test.go | 138 ----------- controllers/druid/testdata/druid-test-cr.yaml | 6 - controllers/druid/volume_expansion.go | 49 ++++ docs/api_specifications/druid.md | 264 +++++---------------- docs/druid_cr.md | 47 ---- docs/features.md | 213 ----------------- e2e/configs/druid-cr.yaml | 11 +- e2e/e2e.sh | 2 +- examples/kafka-ingestion-native.yaml | 68 ------ examples/tiny-cluster-mmless.yaml | 8 +- go.mod | 1 - go.sum | 1 - pkg/druidapi/druidapi.go | 34 +-- pkg/druidapi/druidapi_test.go | 110 --------- pkg/util/util_test.go | 30 --- 25 files changed, 127 insertions(+), 1097 deletions(-) diff --git a/README.md b/README.md index 2c6d790..be95357 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,12 @@ ### Kubernetes version compatibility -| druid-operator | 0.0.9 | v1.0.0 | v1.1.0 | v1.2.2 | v1.2.3 | v1.2.4 | v1.2.5 | v1.3.0 | -| :------------- | :-------------: | :-----: | :---: | :---: | :---: | :---: | :---: | :---: | -| kubernetes <= 1.20 | :x:| :x: | :x: | :x: | :x: | :x: | :x: | :x: | -| kubernetes == 1.21 | :white_check_mark:| :x: | :x: | :x: | :x: | :x: | :x: | :x: | -| kubernetes >= 1.22 and <= 1.25 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| kubernetes > 1.25 and <= 1.33.1 | :x: | :x: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| druid-operator | 0.0.9 | v1.0.0 | v1.1.0 | v1.2.2 | v1.2.3 | +| :------------- | :-------------: | :-----: | :---: | :---: | :---: | +| kubernetes <= 1.20 | :x:| :x: | :x: | :x: | :x: | +| kubernetes == 1.21 | :white_check_mark:| :x: | :x: | :x: | :x: | +| kubernetes >= 1.22 and <= 1.25 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| kubernetes > 1.25 and <= 1.29.1 | :x: | :x: | :white_check_mark: | :white_check_mark: | :white_check_mark: | ### Contributors diff --git a/apis/druid/v1alpha1/druid_types.go b/apis/druid/v1alpha1/druid_types.go index 0e97ae6..0fd457c 100644 --- a/apis/druid/v1alpha1/druid_types.go +++ b/apis/druid/v1alpha1/druid_types.go @@ -290,14 +290,6 @@ type DruidSpec struct { // +optional Auth druidapi.Auth `json:"auth,omitempty"` - - // See v1.DNSPolicy for more details. - // +optional - DNSPolicy v1.DNSPolicy `json:"dnsPolicy,omitempty" protobuf:"bytes,6,opt,name=dnsPolicy,casttype=DNSPolicy"` - - // See v1.PodDNSConfig for more details. - // +optional - DNSConfig *v1.PodDNSConfig `json:"dnsConfig,omitempty" protobuf:"bytes,26,opt,name=dnsConfig"` } // DruidNodeSpec Specification of `Druid` Node type and its configurations. @@ -502,14 +494,6 @@ type DruidNodeSpec struct { // Dynamic Configurations for Druid. Applied through the dynamic configuration API. // +optional DynamicConfig runtime.RawExtension `json:"dynamicConfig,omitempty"` - - // See v1.DNSPolicy for more details. - // +optional - DNSPolicy v1.DNSPolicy `json:"dnsPolicy,omitempty" protobuf:"bytes,6,opt,name=dnsPolicy,casttype=DNSPolicy"` - - // See v1.PodDNSConfig for more details. - // +optional - DNSConfig *v1.PodDNSConfig `json:"dnsConfig,omitempty" protobuf:"bytes,26,opt,name=dnsConfig"` } // ZookeeperSpec IGNORED (Future API): In order to make Druid dependency setup extensible from within Druid operator. diff --git a/apis/druid/v1alpha1/zz_generated.deepcopy.go b/apis/druid/v1alpha1/zz_generated.deepcopy.go index ea1550a..2625397 100644 --- a/apis/druid/v1alpha1/zz_generated.deepcopy.go +++ b/apis/druid/v1alpha1/zz_generated.deepcopy.go @@ -511,11 +511,6 @@ func (in *DruidNodeSpec) DeepCopyInto(out *DruidNodeSpec) { } } in.DynamicConfig.DeepCopyInto(&out.DynamicConfig) - if in.DNSConfig != nil { - in, out := &in.DNSConfig, &out.DNSConfig - *out = new(v1.PodDNSConfig) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DruidNodeSpec. @@ -705,11 +700,6 @@ func (in *DruidSpec) DeepCopyInto(out *DruidSpec) { } in.DynamicConfig.DeepCopyInto(&out.DynamicConfig) out.Auth = in.Auth - if in.DNSConfig != nil { - in, out := &in.DNSConfig, &out.DNSConfig - *out = new(v1.PodDNSConfig) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DruidSpec. diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 6732959..5b95c45 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -15,11 +15,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.9 +version: 0.3.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: v1.3.0 +appVersion: v1.2.5 # icon icon: "https://www.apache.org/logos/res/druid/druid-1.png"; diff --git a/chart/crds/druid.apache.org_druidingestions.yaml b/chart/crds/druid.apache.org_druidingestions.yaml index b5f09c5..70e7fe4 100644 --- a/chart/crds/druid.apache.org_druidingestions.yaml +++ b/chart/crds/druid.apache.org_druidingestions.yaml @@ -47,10 +47,6 @@ spec: properties: auth: properties: - passwordKey: - description: PasswordKey specifies the key within the Kubernetes - secret that contains the password for authentication. - type: string secretRef: description: |- SecretReference represents a Secret Reference. It has enough information to retrieve secret @@ -68,10 +64,6 @@ spec: x-kubernetes-map-type: atomic type: type: string - usernameKey: - description: UsernameKey specifies the key within the Kubernetes - secret that contains the username for authentication. - type: string required: - secretRef - type diff --git a/chart/crds/druid.apache.org_druids.yaml b/chart/crds/druid.apache.org_druids.yaml index 20d7d37..742756d 100644 --- a/chart/crds/druid.apache.org_druids.yaml +++ b/chart/crds/druid.apache.org_druids.yaml @@ -1267,10 +1267,6 @@ spec: type: object auth: properties: - passwordKey: - description: PasswordKey specifies the key within the Kubernetes - secret that contains the password for authentication. - type: string secretRef: description: |- SecretReference represents a Secret Reference. It has enough information to retrieve secret @@ -1288,10 +1284,6 @@ spec: x-kubernetes-map-type: atomic type: type: string - usernameKey: - description: UsernameKey specifies the key within the Kubernetes - secret that contains the username for authentication. - type: string required: - secretRef - type @@ -1509,46 +1501,6 @@ spec: description: DisablePVCDeletionFinalizer Whether PVCs shall be deleted on the deletion of the Druid cluster. type: boolean - dnsConfig: - description: See v1.PodDNSConfig for more details. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options - of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: See v1.DNSPolicy for more details. - type: string dynamicConfig: description: Dynamic Configurations for Druid. Applied through the dynamic configuration API. @@ -3415,46 +3367,6 @@ spec: type: string type: object type: object - dnsConfig: - description: See v1.PodDNSConfig for more details. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options - of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: See v1.DNSPolicy for more details. - type: string druid.port: description: DruidPort Used by the `Druid` process. format: int32 diff --git a/config/crd/bases/druid.apache.org_druidingestions.yaml b/config/crd/bases/druid.apache.org_druidingestions.yaml index b5f09c5..70e7fe4 100644 --- a/config/crd/bases/druid.apache.org_druidingestions.yaml +++ b/config/crd/bases/druid.apache.org_druidingestions.yaml @@ -47,10 +47,6 @@ spec: properties: auth: properties: - passwordKey: - description: PasswordKey specifies the key within the Kubernetes - secret that contains the password for authentication. - type: string secretRef: description: |- SecretReference represents a Secret Reference. It has enough information to retrieve secret @@ -68,10 +64,6 @@ spec: x-kubernetes-map-type: atomic type: type: string - usernameKey: - description: UsernameKey specifies the key within the Kubernetes - secret that contains the username for authentication. - type: string required: - secretRef - type diff --git a/config/crd/bases/druid.apache.org_druids.yaml b/config/crd/bases/druid.apache.org_druids.yaml index 20d7d37..742756d 100644 --- a/config/crd/bases/druid.apache.org_druids.yaml +++ b/config/crd/bases/druid.apache.org_druids.yaml @@ -1267,10 +1267,6 @@ spec: type: object auth: properties: - passwordKey: - description: PasswordKey specifies the key within the Kubernetes - secret that contains the password for authentication. - type: string secretRef: description: |- SecretReference represents a Secret Reference. It has enough information to retrieve secret @@ -1288,10 +1284,6 @@ spec: x-kubernetes-map-type: atomic type: type: string - usernameKey: - description: UsernameKey specifies the key within the Kubernetes - secret that contains the username for authentication. - type: string required: - secretRef - type @@ -1509,46 +1501,6 @@ spec: description: DisablePVCDeletionFinalizer Whether PVCs shall be deleted on the deletion of the Druid cluster. type: boolean - dnsConfig: - description: See v1.PodDNSConfig for more details. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options - of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: See v1.DNSPolicy for more details. - type: string dynamicConfig: description: Dynamic Configurations for Druid. Applied through the dynamic configuration API. @@ -3415,46 +3367,6 @@ spec: type: string type: object type: object - dnsConfig: - description: See v1.PodDNSConfig for more details. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options - of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: See v1.DNSPolicy for more details. - type: string druid.port: description: DruidPort Used by the `Druid` process. format: int32 diff --git a/controllers/druid/druid_controller.go b/controllers/druid/druid_controller.go index 4959d00..1cd6073 100644 --- a/controllers/druid/druid_controller.go +++ b/controllers/druid/druid_controller.go @@ -78,9 +78,9 @@ func (r *DruidReconciler) Reconcile(ctx context.Context, request reconcile.Reque } // Update Druid Dynamic Configs - if err := updateDruidDynamicConfigs(ctx, r.Client, instance, emitEvent); err != nil { - return ctrl.Result{}, err - } + // if err := updateDruidDynamicConfigs(ctx, r.Client, instance, emitEvent); err != nil { + // return ctrl.Result{}, err + // } // If both operations succeed, requeue after specified wait time return ctrl.Result{RequeueAfter: r.ReconcileWait}, nil diff --git a/controllers/druid/handler.go b/controllers/druid/handler.go index 8fc251d..c47032c 100644 --- a/controllers/druid/handler.go +++ b/controllers/druid/handler.go @@ -1122,8 +1122,6 @@ func makePodSpec(nodeSpec *v1alpha1.DruidNodeSpec, m *v1alpha1.Druid, nodeSpecUn SecurityContext: firstNonNilValue(nodeSpec.PodSecurityContext, m.Spec.PodSecurityContext).(*v1.PodSecurityContext), ServiceAccountName: firstNonEmptyStr(nodeSpec.ServiceAccountName, m.Spec.ServiceAccount), PriorityClassName: firstNonEmptyStr(nodeSpec.PriorityClassName, m.Spec.PriorityClassName), - DNSPolicy: v1.DNSPolicy(firstNonEmptyStr(string(nodeSpec.DNSPolicy), string(m.Spec.DNSPolicy))), - DNSConfig: firstNonNilValue(nodeSpec.DNSConfig, m.Spec.DNSConfig).(*v1.PodDNSConfig), } addAdditionalContainers(m, nodeSpec, &spec) diff --git a/controllers/druid/handler_test.go b/controllers/druid/handler_test.go index 55887c9..bde1a87 100644 --- a/controllers/druid/handler_test.go +++ b/controllers/druid/handler_test.go @@ -2,8 +2,6 @@ package druid import ( "io/ioutil" - "reflect" - "testing" "github.com/ghodss/yaml" . "github.com/onsi/ginkgo/v2" @@ -215,139 +213,3 @@ func readAndUnmarshallResource(file string, res interface{}) error { } return nil } - -func TestPodSpecDNSConfig(t *testing.T) { - tests := []struct { - name string - nodeDNSConfig *corev1.PodDNSConfig - specDNSConfig *corev1.PodDNSConfig - expected *corev1.PodDNSConfig - }{ - { - name: "Both nil", - nodeDNSConfig: nil, - specDNSConfig: nil, - expected: nil, - }, - { - name: "Only spec provided", - nodeDNSConfig: nil, - specDNSConfig: &corev1.PodDNSConfig{ - Nameservers: []string{"8.8.8.8"}, - Searches: []string{"example.com"}, - }, - expected: &corev1.PodDNSConfig{ - Nameservers: []string{"8.8.8.8"}, - Searches: []string{"example.com"}, - }, - }, - { - name: "Only node provided", - nodeDNSConfig: &corev1.PodDNSConfig{ - Nameservers: []string{"1.1.1.1"}, - Searches: []string{"node.local"}, - }, - specDNSConfig: nil, - expected: &corev1.PodDNSConfig{ - Nameservers: []string{"1.1.1.1"}, - Searches: []string{"node.local"}, - }, - }, - { - name: "Both provided, node wins", - nodeDNSConfig: &corev1.PodDNSConfig{ - Nameservers: []string{"1.1.1.1"}, - Searches: []string{"node.local"}, - }, - specDNSConfig: &corev1.PodDNSConfig{ - Nameservers: []string{"8.8.8.8"}, - Searches: []string{"example.com"}, - }, - expected: &corev1.PodDNSConfig{ - Nameservers: []string{"1.1.1.1"}, - Searches: []string{"node.local"}, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - m := &druidv1alpha1.Druid{ - Spec: druidv1alpha1.DruidSpec{ - DNSConfig: tc.specDNSConfig, - }, - } - nodeSpec := &druidv1alpha1.DruidNodeSpec{ - DNSConfig: tc.nodeDNSConfig, - } - podSpec := makePodSpec(nodeSpec, m, "unique", "dummySHA") - if !reflect.DeepEqual(podSpec.DNSConfig, tc.expected) { - t.Errorf("expected DNSConfig %v, got %v", tc.expected, podSpec.DNSConfig) - } - }) - } -} - -func TestPodSpecDNSConfigYAML(t *testing.T) { - m, err := readDruidClusterSpecFromFile("testdata/druid-test-cr.yaml") - if err != nil { - t.Fatalf("failed to read cluster spec: %v", err) - } - nodeSpec := m.Spec.Nodes["middlemanagers"] - podSpec := makePodSpec(&nodeSpec, m, "unique", "dummySHA") - expectedDNSConfig := &corev1.PodDNSConfig{ - Nameservers: []string{"10.0.0.53"}, - Searches: []string{"example.local"}, - } - if !reflect.DeepEqual(podSpec.DNSConfig, expectedDNSConfig) { - t.Errorf("expected DNSConfig %v, got %v", expectedDNSConfig, podSpec.DNSConfig) - } -} - -// TestPodSpecDNSPolicy verifies DNSPolicy resolution in makePodSpec. -func TestPodSpecDNSPolicy(t *testing.T) { - tests := []struct { - name string - nodeDNS string - specDNS string - expected corev1.DNSPolicy - }{ - {"Both empty", "", "", corev1.DNSPolicy("")}, - {"Only spec provided", "", "ClusterFirst", corev1.DNSPolicy("ClusterFirst")}, - {"Only node provided", "Default", "", corev1.DNSPolicy("Default")}, - {"Both provided, node wins", "Default", "ClusterFirst", corev1.DNSPolicy("Default")}, - } - - for _, tc := range tests { - tc := tc // capture current test case - t.Run(tc.name, func(t *testing.T) { - m := &druidv1alpha1.Druid{ - Spec: druidv1alpha1.DruidSpec{ - DNSPolicy: corev1.DNSPolicy(tc.specDNS), - }, - } - nodeSpec := &druidv1alpha1.DruidNodeSpec{ - DNSPolicy: corev1.DNSPolicy(tc.nodeDNS), - } - podSpec := makePodSpec(nodeSpec, m, "unique", "dummySHA") - if podSpec.DNSPolicy != tc.expected { - t.Errorf("expected DNSPolicy %q, got %q", tc.expected, podSpec.DNSPolicy) - } - }) - } -} - -// TestPodSpecDNSPolicyYAML validates that the generated PodSpec DNSPolicy matches the expected value, -// using the druid-test-cr.yaml as the single input file. -func TestPodSpecDNSPolicyYAML(t *testing.T) { - m, err := readDruidClusterSpecFromFile("testdata/druid-test-cr.yaml") - if err != nil { - t.Fatalf("failed to read cluster spec: %v", err) - } - nodeSpec := m.Spec.Nodes["middlemanagers"] - podSpec := makePodSpec(&nodeSpec, m, "unique", "dummySHA") - expectedDNSPolicy := corev1.DNSPolicy("ClusterFirst") - if podSpec.DNSPolicy != expectedDNSPolicy { - t.Errorf("expected DNSPolicy %q, got %q", expectedDNSPolicy, podSpec.DNSPolicy) - } -} diff --git a/controllers/druid/testdata/druid-test-cr.yaml b/controllers/druid/testdata/druid-test-cr.yaml index 0cdc8a2..d319740 100644 --- a/controllers/druid/testdata/druid-test-cr.yaml +++ b/controllers/druid/testdata/druid-test-cr.yaml @@ -355,9 +355,3 @@ spec: limits: memory: "3Gi" cpu: "4" - dnsPolicy: ClusterFirst - dnsConfig: - nameservers: - - 10.0.0.53 - searches: - - example.local diff --git a/controllers/druid/volume_expansion.go b/controllers/druid/volume_expansion.go index 3ae6ee0..a98c103 100644 --- a/controllers/druid/volume_expansion.go +++ b/controllers/druid/volume_expansion.go @@ -123,6 +123,13 @@ func scalePVCForSts(ctx context.Context, sdk client.Client, nodeSpec *v1alpha1.D desVolumeClaimTemplateSize, currVolumeClaimTemplateSize, pvcSize := getVolumeClaimTemplateSizes(sts, nodeSpec, pvcList) + // Log the sizes for debugging + fmt.Println("PVC Volume Expansion Debug - Size Arrays") + fmt.Println("StatefulSet:", sts.(*appsv1.StatefulSet).Name) + fmt.Println("desVolumeClaimTemplateSize count:", len(desVolumeClaimTemplateSize)) + fmt.Println("currVolumeClaimTemplateSize count:", len(currVolumeClaimTemplateSize)) + fmt.Println("pvcSize count:", len(pvcSize)) + // current number of PVC can't be less than desired number of pvc if len(pvcSize) < len(desVolumeClaimTemplateSize) { return nil @@ -138,6 +145,15 @@ func scalePVCForSts(ctx context.Context, sdk client.Client, nodeSpec *v1alpha1.D desiredSize, _ := desVolumeClaimTemplateSize[i].AsInt64() currentSize, _ := currVolumeClaimTemplateSize[i].AsInt64() + // Log the raw quantity values for debugging + fmt.Println("\nPVC Volume Expansion Debug - Comparing sizes") + fmt.Println("StatefulSet:", sts.(*appsv1.StatefulSet).Name) + fmt.Println("VolumeClaimTemplate index:", i) + fmt.Println("desiredSize (AsInt64):", desiredSize) + fmt.Println("desiredSize (String):", desVolumeClaimTemplateSize[i].String()) + fmt.Println("currentSize (AsInt64):", currentSize) + fmt.Println("currentSize (String):", currVolumeClaimTemplateSize[i].String()) + if desiredSize < currentSize { e := fmt.Errorf("Request for Shrinking of sts pvc size [sts:%s] in [namespace:%s] is not Supported", sts.(*appsv1.StatefulSet).Name, sts.(*appsv1.StatefulSet).Namespace) logger.Error(e, e.Error(), "name", drd.Name, "namespace", drd.Namespace) @@ -165,11 +181,44 @@ func scalePVCForSts(ctx context.Context, sdk client.Client, nodeSpec *v1alpha1.D // In case size dont match, patch the pvc with the desiredsize from druid CR for p := range pvcSize { pSize, _ := pvcSize[p].AsInt64() + + // Log PVC comparison details + pvcObj := pvcList[p].(*v1.PersistentVolumeClaim) + fmt.Println("\nPVC Volume Expansion Debug - PVC comparison") + fmt.Println("PVC:", pvcObj.Name) + fmt.Println("desiredSize (AsInt64):", desiredSize) + fmt.Println("desiredSize (String):", desVolumeClaimTemplateSize[i].String()) + fmt.Println("pvcSize (AsInt64):", pSize) + fmt.Println("pvcSize (String):", pvcSize[p].String()) + + specStorage := "nil" + if reqStorage, exists := pvcObj.Spec.Resources.Requests[v1.ResourceStorage]; exists { + specStorage = reqStorage.String() + } + fmt.Println("PVC spec.resources.requests.storage:", specStorage) + + statusStorage := "nil" + if pvcObj.Status.Capacity != nil { + if cap, exists := pvcObj.Status.Capacity[v1.ResourceStorage]; exists { + statusStorage = cap.String() + } + } + fmt.Println("PVC status.capacity.storage:", statusStorage) + fmt.Println("sizes equal?:", desiredSize == pSize) + if desiredSize != pSize { + fmt.Println("\nPVC Volume Expansion Debug - Attempting to patch PVC") + fmt.Println("PVC:", pvcObj.Name) + fmt.Println("from size:", pvcSize[p].String()) + fmt.Println("to size:", desVolumeClaimTemplateSize[i].String()) + // use deepcopy patch := client.MergeFrom(pvcList[p].(*v1.PersistentVolumeClaim).DeepCopy()) pvcList[p].(*v1.PersistentVolumeClaim).Spec.Resources.Requests[v1.ResourceStorage] = desVolumeClaimTemplateSize[i] if err := writers.Patch(ctx, sdk, drd, pvcList[p].(*v1.PersistentVolumeClaim), false, patch, emitEvent); err != nil { + fmt.Println("\nPVC Volume Expansion Debug - Patch failed") + fmt.Println("PVC:", pvcObj.Name) + fmt.Println("error:", err.Error()) return err } else { msg := fmt.Sprintf("[PVC:%s] successfully Patched with [Size:%s]", pvcList[p].(*v1.PersistentVolumeClaim).Name, desVolumeClaimTemplateSize[i].String()) diff --git a/docs/api_specifications/druid.md b/docs/api_specifications/druid.md index 4517835..fd57ded 100644 --- a/docs/api_specifications/druid.md +++ b/docs/api_specifications/druid.md @@ -173,6 +173,56 @@ Kubernetes core/v1.ResourceRequirements </table> </div> </div> +<h3 id="druid.apache.org/v1alpha1.Auth">Auth +</h3> +<p> +(<em>Appears on:</em> +<a href="#druid.apache.org/v1alpha1.DruidIngestionSpec">DruidIngestionSpec</a>) +</p> +<div class="md-typeset__scrollwrap"> +<div class="md-typeset__table"> +<table> +<thead> +<tr> +<th>Field</th> +<th>Description</th> +</tr> +</thead> +<tbody> +<tr> +<td> +<code>type</code><br> +<em> +<a href="#druid.apache.org/v1alpha1.AuthType"> +AuthType +</a> +</em> +</td> +<td> +</td> +</tr> +<tr> +<td> +<code>secretRef</code><br> +<em> +<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#secretreference-v1-core";> +Kubernetes core/v1.SecretReference +</a> +</em> +</td> +<td> +</td> +</tr> +</tbody> +</table> +</div> +</div> +<h3 id="druid.apache.org/v1alpha1.AuthType">AuthType +(<code>string</code> alias)</h3> +<p> +(<em>Appears on:</em> +<a href="#druid.apache.org/v1alpha1.Auth">Auth</a>) +</p> <h3 id="druid.apache.org/v1alpha1.DeepStorageSpec">DeepStorageSpec </h3> <p> @@ -857,57 +907,6 @@ string <p>CoreSite Contents of <code>core-site.xml</code>.</p> </td> </tr> -<tr> -<td> -<code>dynamicConfig</code><br> -<em> -k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -<p>Dynamic Configurations for Druid. Applied through the dynamic configuration API.</p> -</td> -</tr> -<tr> -<td> -<code>auth</code><br> -<em> -github.com/datainfrahq/druid-operator/pkg/druidapi.Auth -</em> -</td> -<td> -<em>(Optional)</em> -</td> -</tr> -<tr> -<td> -<code>dnsPolicy</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#dnspolicy-v1-core";> -Kubernetes core/v1.DNSPolicy -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.DNSPolicy for more details.</p> -</td> -</tr> -<tr> -<td> -<code>dnsConfig</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#poddnsconfig-v1-core";> -Kubernetes core/v1.PodDNSConfig -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.PodDNSConfig for more details.</p> -</td> -</tr> </table> </td> </tr> @@ -1129,7 +1128,9 @@ IngestionSpec <td> <code>auth</code><br> <em> -github.com/datainfrahq/druid-operator/pkg/druidapi.Auth +<a href="#druid.apache.org/v1alpha1.Auth"> +Auth +</a> </em> </td> <td> @@ -1214,7 +1215,9 @@ IngestionSpec <td> <code>auth</code><br> <em> -github.com/datainfrahq/druid-operator/pkg/druidapi.Auth +<a href="#druid.apache.org/v1alpha1.Auth"> +Auth +</a> </em> </td> <td> @@ -1313,18 +1316,6 @@ string </em> </td> <td> -<p>CurrentIngestionSpec is a string instead of RawExtension to maintain compatibility with existing -IngestionSpecs that are stored as JSON strings.</p> -</td> -</tr> -<tr> -<td> -<code>rules</code><br> -<em> -[]k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> </td> </tr> </tbody> @@ -1951,58 +1942,6 @@ Kubernetes autoscaling/v2.HorizontalPodAutoscalerSpec <p>Operator deploys the sidecar container based on these properties.</p> </td> </tr> -<tr> -<td> -<code>serviceAccountName</code><br> -<em> -string -</em> -</td> -<td> -<em>(Optional)</em> -<p>ServiceAccountName Kubernetes native <code>serviceAccountName</code> specification.</p> -</td> -</tr> -<tr> -<td> -<code>dynamicConfig</code><br> -<em> -k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -<p>Dynamic Configurations for Druid. Applied through the dynamic configuration API.</p> -</td> -</tr> -<tr> -<td> -<code>dnsPolicy</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#dnspolicy-v1-core";> -Kubernetes core/v1.DNSPolicy -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.DNSPolicy for more details.</p> -</td> -</tr> -<tr> -<td> -<code>dnsConfig</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#poddnsconfig-v1-core";> -Kubernetes core/v1.PodDNSConfig -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.PodDNSConfig for more details.</p> -</td> -</tr> </tbody> </table> </div> @@ -2687,57 +2626,6 @@ string <p>CoreSite Contents of <code>core-site.xml</code>.</p> </td> </tr> -<tr> -<td> -<code>dynamicConfig</code><br> -<em> -k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -<p>Dynamic Configurations for Druid. Applied through the dynamic configuration API.</p> -</td> -</tr> -<tr> -<td> -<code>auth</code><br> -<em> -github.com/datainfrahq/druid-operator/pkg/druidapi.Auth -</em> -</td> -<td> -<em>(Optional)</em> -</td> -</tr> -<tr> -<td> -<code>dnsPolicy</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#dnspolicy-v1-core";> -Kubernetes core/v1.DNSPolicy -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.DNSPolicy for more details.</p> -</td> -</tr> -<tr> -<td> -<code>dnsConfig</code><br> -<em> -<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#poddnsconfig-v1-core";> -Kubernetes core/v1.PodDNSConfig -</a> -</em> -</td> -<td> -<em>(Optional)</em> -<p>See v1.PodDNSConfig for more details.</p> -</td> -</tr> </tbody> </table> </div> @@ -2778,52 +2666,12 @@ string </em> </td> <td> -<em>(Optional)</em> -<p>Spec should be passed in as a JSON string. -Note: This field is planned for deprecation in favor of nativeSpec.</p> <br/> <br/> <table> </table> </td> </tr> -<tr> -<td> -<code>nativeSpec</code><br> -<em> -k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -<p>nativeSpec allows the ingestion specification to be defined in a native Kubernetes format. -This is particularly useful for environment-specific configurations and will eventually -replace the JSON-based Spec field. -Note: Spec will be ignored if nativeSpec is provided.</p> -</td> -</tr> -<tr> -<td> -<code>compaction</code><br> -<em> -k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -</td> -</tr> -<tr> -<td> -<code>rules</code><br> -<em> -[]k8s.io/apimachinery/pkg/runtime.RawExtension -</em> -</td> -<td> -<em>(Optional)</em> -</td> -</tr> </tbody> </table> </div> diff --git a/docs/druid_cr.md b/docs/druid_cr.md index 5d056d4..176e81a 100644 --- a/docs/druid_cr.md +++ b/docs/druid_cr.md @@ -68,50 +68,3 @@ spec: runtime.properties: | ... ``` - -## Authentication Setup - -Authentication can be configured to secure communication with the cluster API using credentials stored in Kubernetes secrets. - -Currently this is used for compaction, rules, dynamic configs, and ingestion configurations. - -This not only applies to the `Druid` CR but also to the `DruidIngestion` CR. - -### Configuring Basic Authentication - -To use basic authentication, you need to create a Kubernetes secret containing the username and password. This secret is then referenced in the Druid CR. - -Steps to Configure Basic Authentication: - -1. **Create a Kubernetes Secret:** Store your username and password in a Kubernetes secret. Below is an example of how to define the secret in a YAML file: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: mycluster-admin-operator - namespace: druid -type: Opaque -data: - OperatorUserName: <base64-encoded-username> - OperatorPassword: <base64-encoded-password> -``` - -Replace <base64-encoded-username> and <base64-encoded-password> with the base64-encoded values of your desired username and password. - -2. Define Authentication in the Druid CRD: Reference the secret in your Druid custom resource. Here is an example `Druid`: - -```yaml -apiVersion: druid.apache.org/v1alpha1 -kind: Druid -metadata: - name: agent -spec: - auth: - secretRef: - name: mycluster-admin-operator - namespace: druid - type: basic-auth -``` - -This configuration specifies that the Druid cluster should use basic authentication with credentials retrieved from the mycluster-admin-operator secret. diff --git a/docs/features.md b/docs/features.md index 18eac20..4038b6c 100644 --- a/docs/features.md +++ b/docs/features.md @@ -225,216 +225,3 @@ All the probes definitions are documented bellow: ``` </details> - -## Dynamic Configurations - -The Druid operator now supports specifying dynamic configurations directly within the Druid manifest. This feature allows for fine-tuned control over Druid's behavior at runtime by adjusting configurations dynamically. - - -### Overlord Dynamic Configurations - -Usage: Add overlord dynamic configurations under the middlemanagers section within the nodes element of the Druid manifest. - -<details> - -<summary>Overlord Dynamic Configurations</summary> - -```yaml -spec: - nodes: - middlemanagers: - dynamicConfig: - type: default - selectStrategy: - type: fillCapacityWithCategorySpec - workerCategorySpec: - categoryMap: {} - strong: true - autoScaler: null -``` - -</details> - -### Coordinator Dynamic Configurations - -Adjust coordinator settings to optimize data balancing and segment management. - -Usage: Include coordinator dynamic configurations in the coordinator section within the nodes element of the Druid manifest. - -Ensure all parameters are supported for the operator to properly configure dynamic configurations. - -<details> - -<summary>Overlord Dynamic Configurations</summary> - -```yaml -spec: - nodes: - coordinators: - dynamicConfig: - millisToWaitBeforeDeleting: 900000 - mergeBytesLimit: 524288000 - mergeSegmentsLimit: 100 - maxSegmentsToMove: 5 - replicantLifetime: 15 - replicationThrottleLimit: 10 - balancerComputeThreads: 1 - killDataSourceWhitelist: [] - killPendingSegmentsSkipList: [] - maxSegmentsInNodeLoadingQueue: 100 - decommissioningNodes: [] - pauseCoordination: false - replicateAfterLoadTimeout: false - useRoundRobinSegmentAssignment: true -``` - -</details> - -## nativeSpec Ingestion Configuration - -The `nativeSpec` feature in the Druid Ingestion Operator provides a flexible and robust way to define ingestion specifications directly within Kubernetes manifests using YAML format. This enhancement allows users to leverage Kubernetes-native formats, facilitating easier integration with Kubernetes tooling and practices while offering a more readable and maintainable configuration structure. - -### Key Benefits - -* **Kubernetes-Native Integration:** By using YAML, the `nativeSpec` aligns with Kubernetes standards, enabling seamless integration with Kubernetes-native tools and processes, such as kubectl, Helm, and GitOps workflows. -* **Improved Readability and Maintainability:** YAML's human-readable format makes it easier for operators and developers to understand and modify ingestion configurations without deep JSON knowledge or tools. -* **Enhanced Configuration Management:** Leveraging YAML facilitates the use of environment-specific configurations and overrides, making it easier to manage configurations across different stages of deployment (e.g., development, staging, production). - -### Usage - -Specifying nativeSpec in Kubernetes Manifests - -To use `nativeSpec`, define your ingestion specifications in YAML format under the `nativeSpec` field in the Druid Ingestion Custom Resource Definition (CRD). This field supercedes the traditional JSON `spec` field, providing a more integrated approach to configuration management. - -<details> - -<summary>nativeSpec Example</summary> - -```yaml -apiVersion: druid.apache.org/v1alpha1 -kind: DruidIngestion -metadata: - labels: - app.kubernetes.io/name: druidingestion - app.kubernetes.io/instance: druidingestion-sample - name: kafka-1 -spec: - suspend: false - druidCluster: example-cluster - ingestion: - type: kafka - nativeSpec: - type: kafka - spec: - dataSchema: - dataSource: metrics-kafka-1 - timestampSpec: - column: timestamp - format: auto - dimensionsSpec: - dimensions: [] - dimensionExclusions: - - timestamp - - value - metricsSpec: - - name: count - type: count - - name: value_sum - fieldName: value - type: doubleSum - - name: value_min - fieldName: value - type: doubleMin - - name: value_max - fieldName: value - type: doubleMax - granularitySpec: - type: uniform - segmentGranularity: HOUR - queryGranularity: NONE - ioConfig: - topic: metrics - inputFormat: - type: json - consumerProperties: - bootstrap.servers: localhost:9092 - taskCount: 1 - replicas: 1 - taskDuration: PT1H - tuningConfig: - type: kafka - maxRowsPerSegment: 5000000 - -``` - -</details> - -## Set Rules and Compaction in DruidIngestion - -### Rules - -Rules in Druid define automated behaviors such as data retention, load balancing, or replication. They can be configured in the Rules section of the `DruidIngestion` CRD. - -<details> - -<summary>Rules Example</summary> - -```yaml -apiVersion: druid.apache.org/v1alpha1 -kind: DruidIngestion -metadata: - name: example-druid-ingestion -spec: - ingestion: - type: native-batch - rules: - - type: "loadForever" - tieredReplicants: - _default_tier: 2 - - type: "dropByPeriod" - period: "P7D" -``` - -</details> - -### Compaction - -Compaction in Druid helps optimize data storage and query performance by merging smaller data segments into larger ones. The compaction configuration can be specified in the Compaction section of the DruidIngestion CRD. - -The Druid Operator ensures accurate application of compaction settings by: - -1. Retrieving Current Settings: It performs a GET request on the Druid API to fetch existing compaction settings. - -2. Comparing and Updating: If there is a discrepancy between current and desired settings specified in the Kubernetes CRD manifest, the operator updates Druid with the desired configuration. - -3. Ensuring Accuracy: This method ensures settings are correctly applied, addressing cases where Druid might return a 200 HTTP status code without saving the changes. - -<details> - -<summary>Compaction Example</summary> - -```yaml -apiVersion: druid.apache.org/v1alpha1 -kind: DruidIngestion -metadata: - name: example-druid-ingestion -spec: - ingestion: - type: native-batch - compaction: - ioConfig: - type: "index_parallel" - inputSpec: - type: "dataSource" - dataSource: "my-data-source" - tuningConfig: - maxNumConcurrentSubTasks: 4 - granularitySpec: - segmentGranularity: "day" - queryGranularity: "none" - rollup: false - taskPriority: "high" - taskContext: '{"priority": 75}' -``` - -</details> diff --git a/e2e/configs/druid-cr.yaml b/e2e/configs/druid-cr.yaml index 0fff78c..69cbe4d 100644 --- a/e2e/configs/druid-cr.yaml +++ b/e2e/configs/druid-cr.yaml @@ -121,12 +121,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - dnsPolicy: ClusterFirst - dnsConfig: - nameservers: - - 10.0.0.53 - searches: - - example.local nodes: brokers: # Optionally specify for running broker as Deployment @@ -195,15 +189,20 @@ spec: mergeBytesLimit: 524288000 mergeSegmentsLimit: 100 maxSegmentsToMove: 5 + percentOfSegmentsToConsiderPerMove: 100 + useBatchedSegmentSampler: true replicantLifetime: 15 replicationThrottleLimit: 10 balancerComputeThreads: 1 + emitBalancingStats: true killDataSourceWhitelist: [] killPendingSegmentsSkipList: [] maxSegmentsInNodeLoadingQueue: 100 decommissioningNodes: [] + decommissioningMaxPercentOfMaxSegmentsToMove: 70 pauseCoordination: false replicateAfterLoadTimeout: false + maxNonPrimaryReplicantsToLoad: 2147483647 useRoundRobinSegmentAssignment: true historicals: diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 18b5438..6d9cf2f 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -75,7 +75,7 @@ else echo "Supervisor task ID: $supervisorTaskId" fi -# Running a test Kafka DruidIngestion resource using nativeSpec and wait for the task to be submitted +# Running a test Kafka DruidIngestion resource and wait for the task to be submitted kubectl apply -f e2e/configs/kafka-ingestion-native.yaml -n ${NAMESPACE} sleep 30 # wait for the manager to submit the ingestion task diff --git a/examples/kafka-ingestion-native.yaml b/examples/kafka-ingestion-native.yaml deleted file mode 100644 index 18fff8d..0000000 --- a/examples/kafka-ingestion-native.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: druid.apache.org/v1alpha1 -kind: DruidIngestion -metadata: - labels: - app.kubernetes.io/name: druidingestion - app.kubernetes.io/instance: druidingestion-sample - name: kafka-2 -spec: - suspend: false - druidCluster: tiny-cluster - ingestion: - type: kafka - compaction: - tuningConfig: - type: "kafka" - partitionsSpec: - type: "dynamic" - skipOffsetFromLatest: "PT0S" - granularitySpec: - segmentGranularity: "DAY" - rules: - - type: dropByPeriod - period: P1M - includeFuture: true - - type: broadcastByPeriod - period: P1M - includeFuture: true - nativeSpec: - type: kafka - spec: - dataSchema: - dataSource: metrics-kafka-2 - timestampSpec: - column: timestamp - format: auto - dimensionsSpec: - dimensions: [] - dimensionExclusions: - - timestamp - - value - metricsSpec: - - name: count - type: count - - name: value_sum - fieldName: value - type: doubleSum - - name: value_min - fieldName: value - type: doubleMin - - name: value_max - fieldName: value - type: doubleMax - granularitySpec: - type: uniform - segmentGranularity: HOUR - queryGranularity: NONE - ioConfig: - topic: metrics - inputFormat: - type: json - consumerProperties: - bootstrap.servers: localhost:9092 - taskCount: 1 - replicas: 1 - taskDuration: PT1H - tuningConfig: - type: kafka - maxRowsPerSegment: 5000000 diff --git a/examples/tiny-cluster-mmless.yaml b/examples/tiny-cluster-mmless.yaml index ea654d1..371710d 100644 --- a/examples/tiny-cluster-mmless.yaml +++ b/examples/tiny-cluster-mmless.yaml @@ -48,7 +48,6 @@ spec: commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common" jvm.options: |- -server - -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 @@ -137,7 +136,7 @@ spec: type: ClusterIP clusterIP: None nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker" - replicas: 1 + replicas: s runtime.properties: | druid.service=druid/broker # HTTP server threads @@ -146,7 +145,6 @@ spec: # Processing threads and buffers druid.processing.buffer.sizeBytes=25000000 druid.sql.enable=true - druid.host=druid-tiny-cluster-brokers extra.jvm.options: |- -Xmx512m -Xms512m @@ -175,7 +173,6 @@ spec: druid.indexer.runner.namespace: druid druid.indexer.runner.type: k8s druid.indexer.task.encapsulatedTask: true - druid.host=druid-tiny-cluster-coordinators extra.jvm.options: |- -Xmx800m -Xms800m @@ -242,7 +239,6 @@ spec: # Segment storage druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":1000000000}] druid.server.maxSize=1000000000 - druid.host=druid-tiny-cluster-hot extra.jvm.options: |- -Xmx512m -Xms512m @@ -309,7 +305,6 @@ spec: # Segment storage druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":2000000000}] druid.server.maxSize=2000000000 - druid.host=druid-tiny-cluster-cold extra.jvm.options: |- -Xmx512m -Xms512m @@ -336,7 +331,6 @@ spec: druid.router.coordinatorServiceName=druid/coordinator # Management proxy to coordinator / overlord: required for unified web console. druid.router.managementProxy.enabled=true - druid.host=druid-tiny-cluster-routers --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/go.mod b/go.mod index 01f3448..14ca62f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect diff --git a/go.sum b/go.sum index 43ed29b..76aaa3b 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,6 @@ github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= diff --git a/pkg/druidapi/druidapi.go b/pkg/druidapi/druidapi.go index effda50..c932dcc 100644 --- a/pkg/druidapi/druidapi.go +++ b/pkg/druidapi/druidapi.go @@ -30,12 +30,6 @@ type Auth struct { Type AuthType `json:"type"` // +required SecretRef v1.SecretReference `json:"secretRef"` - - // UsernameKey specifies the key within the Kubernetes secret that contains the username for authentication. - UsernameKey string `json:"usernameKey,omitempty"` - - // PasswordKey specifies the key within the Kubernetes secret that contains the password for authentication. - PasswordKey string `json:"passwordKey,omitempty"` } // GetAuthCreds retrieves basic authentication credentials from a Kubernetes secret. @@ -48,23 +42,12 @@ type Auth struct { // // Returns: // -// BasicAuth: The basic authentication credentials, or an error if authentication retrieval fails. +// BasicAuth: The basic authentication credentials. func GetAuthCreds( ctx context.Context, c client.Client, auth Auth, ) (internalhttp.BasicAuth, error) { - userNameKey := OperatorUserName - passwordKey := OperatorPassword - - if auth.UsernameKey != "" { - userNameKey = auth.UsernameKey - } - - if auth.PasswordKey != "" { - passwordKey = auth.PasswordKey - } - // Check if the mentioned secret exists if auth != (Auth{}) { secret := v1.Secret{} @@ -74,18 +57,9 @@ func GetAuthCreds( }, &secret); err != nil { return internalhttp.BasicAuth{}, err } - - if _, ok := secret.Data[userNameKey]; !ok { - return internalhttp.BasicAuth{}, fmt.Errorf("username key %q not found in secret %s/%s", userNameKey, auth.SecretRef.Namespace, auth.SecretRef.Name) - } - - if _, ok := secret.Data[passwordKey]; !ok { - return internalhttp.BasicAuth{}, fmt.Errorf("password key %q not found in secret %s/%s", passwordKey, auth.SecretRef.Namespace, auth.SecretRef.Name) - } - creds := internalhttp.BasicAuth{ - UserName: string(secret.Data[userNameKey]), - Password: string(secret.Data[passwordKey]), + UserName: string(secret.Data[OperatorUserName]), + Password: string(secret.Data[OperatorPassword]), } return creds, nil @@ -155,7 +129,7 @@ func GetRouterSvcUrl(namespace, druidClusterName string, c client.Client) (strin return "", errors.New("router svc discovery fail") } - newName := "http://"; + svcName + "." + namespace + ":" + DruidRouterPort + newName := "http://"; + svcName + "." + namespace + ".svc.cluster.local:" + DruidRouterPort return newName, nil } diff --git a/pkg/druidapi/druidapi_test.go b/pkg/druidapi/druidapi_test.go index 10a686c..3b94e99 100644 --- a/pkg/druidapi/druidapi_test.go +++ b/pkg/druidapi/druidapi_test.go @@ -1,119 +1,9 @@ package druidapi import ( - "context" - internalhttp "github.com/datainfrahq/druid-operator/pkg/http" - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" ) -func TestGetAuthCreds(t *testing.T) { - tests := []struct { - name string - auth Auth - expected internalhttp.BasicAuth - expectErr bool - }{ - { - name: "default keys present", - auth: Auth{ - Type: BasicAuth, - SecretRef: v1.SecretReference{Name: "test-default", Namespace: "test"}, - }, - expected: internalhttp.BasicAuth{UserName: "test-user", Password: "test-password"}, - expectErr: false, - }, - { - name: "custom keys present", - auth: Auth{ - Type: BasicAuth, - SecretRef: v1.SecretReference{Name: "test", Namespace: "default"}, - UsernameKey: "usr", - PasswordKey: "pwd", - }, - expected: internalhttp.BasicAuth{UserName: "admin", Password: "admin"}, - expectErr: false, - }, - { - name: "custom user key is missing", - auth: Auth{ - Type: BasicAuth, - SecretRef: v1.SecretReference{Name: "test", Namespace: "default"}, - UsernameKey: "nope", - PasswordKey: "pwd", - }, - expected: internalhttp.BasicAuth{}, - expectErr: true, - }, - { - name: "custom user key with default password key", - auth: Auth{ - Type: BasicAuth, - SecretRef: v1.SecretReference{Name: "test", Namespace: "default"}, - UsernameKey: "usr", - }, - expected: internalhttp.BasicAuth{UserName: "admin", Password: "also-admin"}, - expectErr: false, - }, - { - name: "custom password key is missing", - auth: Auth{ - Type: BasicAuth, - SecretRef: v1.SecretReference{Name: "test", Namespace: "default"}, - UsernameKey: "usr", - PasswordKey: "nope", - }, - expected: internalhttp.BasicAuth{}, - expectErr: true, - }, - { - name: "empty auth struct returns no creds", - auth: Auth{}, - expected: internalhttp.BasicAuth{}, - expectErr: false, - }, - } - - client := fake.NewClientBuilder(). - WithObjects(&v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-default", - Namespace: "test", - }, - Data: map[string][]byte{ - OperatorUserName: []byte("test-user"), - OperatorPassword: []byte("test-password"), - }, - }). - WithObjects(&v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Data: map[string][]byte{ - "usr": []byte("admin"), - "pwd": []byte("admin"), - OperatorPassword: []byte("also-admin"), - }, - }).Build() - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actual, err := GetAuthCreds(context.TODO(), client, tt.auth) - if tt.expectErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - - assert.Equal(t, tt.expected, actual) - }) - } -} - func TestMakePath(t *testing.T) { tests := []struct { name string diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 3535e44..30d1c04 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -52,36 +52,6 @@ func TestIncludesJson(t *testing.T) { expectedEqual: false, expectError: false, }, - { - // This covers a case where a user may use the JSON body from the cURL example - // in Druid documentation (https://druid.apache.org/docs/latest/api-reference/dynamic-configuration-api), - // which includes extra fields that are not supported by Druid anymore. - name: "Incorrect coordinator dynamic config", - currentJson: `{"millisToWaitBeforeDeleting":900000,"mergeBytesLimit":524288000,"mergeSegmentsLimit":100,"maxSegmentsToMove":5,"replicantLifetime":15,"replicationThrottleLimit":10,"balancerComputeThreads":1,"killDataSourceWhitelist":[],"killTaskSlotRatio":0.1,"maxKillTaskSlots":2147483647,"killPendingSegmentsSkipList":[],"maxSegmentsInNodeLoadingQueue":100,"decommissioningNodes":[],"pauseCoordination":false,"replicateAfterLoadTimeout":false,"useRoundRobinSegmentAssignment":true,"smartS [...] - desiredJson: `{ - "millisToWaitBeforeDeleting": 900000, - "mergeBytesLimit": 524288000, - "mergeSegmentsLimit": 100, - "maxSegmentsToMove": 5, - "percentOfSegmentsToConsiderPerMove": 100, - "useBatchedSegmentSampler": false, - "replicantLifetime": 15, - "replicationThrottleLimit": 10, - "balancerComputeThreads": 1, - "emitBalancingStats": false, - "killDataSourceWhitelist": [], - "killAllDataSources": true, - "killPendingSegmentsSkipList": [], - "maxSegmentsInNodeLoadingQueue": 100, - "decommissioningNodes": [], - "decommissioningMaxPercentOfMaxSegmentsToMove": 70, - "pauseCoordination": false, - "replicateAfterLoadTimeout": false, - "maxNonPrimaryReplicantsToLoad": 2147483647 - }`, - expectedEqual: false, - expectError: false, - }, { name: "Subset match with nested maps", currentJson: `{ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
![https://www.apache.org/logos/res/druid/druid-1.png"](https://www.apache.org/logos/res/druid/druid-1.png"){kind=link}