Skip to content

Commit

Permalink
introduce tests to check whether workqueue metrics exist
Browse files Browse the repository at this point in the history
Signed-off-by: chaosi-zju <[email protected]>
  • Loading branch information
chaosi-zju committed Jan 10, 2025
1 parent 9f8da2c commit 5462a31
Show file tree
Hide file tree
Showing 6 changed files with 347 additions and 5 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/installation-cli.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
hack/cli-testing-environment.sh
# run a single e2e
export KUBECONFIG=${HOME}/karmada/karmada-apiserver.config
export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/
- name: export logs
Expand Down Expand Up @@ -87,7 +87,7 @@ jobs:
hack/cli-testing-init-with-config.sh
# run a single e2e
export KUBECONFIG=${HOME}/karmada/karmada-apiserver.config
export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
ginkgo -v --race --trace -p --focus="[BasicPropagation] propagation testing deployment propagation testing" ./test/e2e/
- name: export logs for config test
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ require (
github.com/onsi/gomega v1.34.1
github.com/opensearch-project/opensearch-go v1.1.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/common v0.55.0
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
Expand Down Expand Up @@ -134,7 +135,6 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
Expand Down
10 changes: 9 additions & 1 deletion test/e2e/framework/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,15 @@ func WaitClusterFitWith(c client.Client, clusterName string, fit func(cluster *c

// LoadRESTClientConfig creates a rest.Config using the passed kubeconfig. If context is empty, current context in kubeconfig will be used.
func LoadRESTClientConfig(kubeconfig string, context string) (*rest.Config, error) {
loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
var loader *clientcmd.ClientConfigLoadingRules
if strings.Contains(kubeconfig, ":") {
// kubeconfig is a list of kubeconfig files in form of "file1:file2:file3"
loader = &clientcmd.ClientConfigLoadingRules{Precedence: strings.Split(kubeconfig, ":")}
} else {
// kubeconfig is a single file
loader = &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
}

loadedConfig, err := loader.Load()
if err != nil {
return nil, err
Expand Down
114 changes: 114 additions & 0 deletions test/e2e/metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
Copyright 2023 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/util/rand"

policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
"github.com/karmada-io/karmada/test/e2e/framework"
testhelper "github.com/karmada-io/karmada/test/helper"
)

var _ = ginkgo.Describe("metrics testing", func() {
var grabber *testhelper.Grabber

var componentMetrics = map[string][]string{
testhelper.KarmadaControllerManagerDeploy: {
"workqueue_queue_duration_seconds_sum", // workqueue metrics
"cluster_ready_state", // custom ClusterCollectors metrics
"work_sync_workload_duration_seconds_sum", // custom ResourceCollectors metrics
},
testhelper.KarmadaSchedulerDeploy: {
"workqueue_queue_duration_seconds_sum", // workqueue metrics
//"schedule_attempts_total", // scheduler custom metrics
},
testhelper.KarmadaDeschedulerDeploy: {
"workqueue_queue_duration_seconds_sum", // workqueue metrics
},
testhelper.KarmadaMetricsAdapterDeploy: {
"workqueue_queue_duration_seconds_sum", // workqueue metrics
},
testhelper.KarmadaSchedulerEstimatorDeploy: {
"karmada_scheduler_estimator_estimating_request_total", // scheduler estimator custom metrics
},
testhelper.KarmadaWebhookDeploy: {
"controller_runtime_webhook_requests_total", // controller runtime hook server metrics
},
}

ginkgo.BeforeEach(func() {
var err error
grabber, err = testhelper.NewMetricsGrabber(context.TODO(), hostKubeClient)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})

ginkgo.Context("metrics presence testing", func() {
ginkgo.It("metrics presence testing for each component", func() {
ginkgo.By("1. do a simple scheduling to ensure above metrics exist", func() {
name := deploymentNamePrefix + rand.String(RandomStrLength)
deployment := testhelper.NewDeployment(testNamespace, name)
policy := testhelper.NewPropagationPolicy(testNamespace, name, []policyv1alpha1.ResourceSelector{
{
APIVersion: deployment.APIVersion,
Kind: deployment.Kind,
Name: deployment.Name,
},
}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
ClusterNames: framework.ClusterNames(),
},
})
framework.CreateDeployment(kubeClient, deployment)
framework.CreatePropagationPolicy(karmadaClient, policy)
ginkgo.DeferCleanup(func() {
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
})
framework.WaitDeploymentPresentOnClustersFitWith(framework.ClusterNames(), deployment.Namespace, deployment.Name, func(_ *appsv1.Deployment) bool { return true })
})

ginkgo.By("2. judge metrics presence", func() {
for component, metricsList := range componentMetrics {
// the output format of `metrics` is like:
// {
// "workqueue_queue_duration_seconds_sum": [{
// "metric": {
// "__name__": "workqueue_queue_duration_seconds_sum",
// "controller": "work-status-controller",
// "name": "work-status-controller"
// },
// "value": [0, "0.12403110800000001"]
// }]
// }
metrics, err := grabber.GrabMetricsFromComponent(context.TODO(), component)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())

for _, metricName := range metricsList {
testhelper.PrintMetricSample((*metrics)[metricName])
gomega.Expect((*metrics)[metricName]).ShouldNot(gomega.BeNil())
}
}
})
})
})
})
12 changes: 11 additions & 1 deletion test/e2e/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,13 @@ var (
)

var (
hostContext string
karmadaContext string
kubeconfig string
karmadactlPath string
restConfig *rest.Config
karmadaHost string
hostKubeClient kubernetes.Interface
kubeClient kubernetes.Interface
karmadaClient karmada.Interface
dynamicClient dynamic.Interface
Expand All @@ -125,7 +127,8 @@ func init() {
// eg. ginkgo -v --race --trace --fail-fast -p --randomize-all ./test/e2e/ -- --poll-interval=5s --poll-timeout=5m
flag.DurationVar(&pollInterval, "poll-interval", 5*time.Second, "poll-interval defines the interval time for a poll operation")
flag.DurationVar(&pollTimeout, "poll-timeout", 300*time.Second, "poll-timeout defines the time which the poll operation times out")
flag.StringVar(&karmadaContext, "karmada-context", karmadaContext, "Name of the cluster context in control plane kubeconfig file.")
flag.StringVar(&hostContext, "host-context", "karmada-host", "Name of the host cluster context in control plane kubeconfig file.")
flag.StringVar(&karmadaContext, "karmada-context", "karmada-apiserver", "Name of the karmada cluster context in control plane kubeconfig file.")
}

func TestE2E(t *testing.T) {
Expand All @@ -148,6 +151,13 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
gomega.Expect(karmadactlPath).ShouldNot(gomega.BeEmpty())

clusterProvider = cluster.NewProvider()

restConfig, err = framework.LoadRESTClientConfig(kubeconfig, hostContext)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())

hostKubeClient, err = kubernetes.NewForConfig(restConfig)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())

restConfig, err = framework.LoadRESTClientConfig(kubeconfig, karmadaContext)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())

Expand Down
Loading

0 comments on commit 5462a31

Please sign in to comment.