Merge pull request #36 from Azure/custom-metrics-config

Custom metrics advanced config file and fixes scale down issue

Gopkg.toml

@@ -7,13 +7,19 @@ ignored = ["github.com/Azure/azure-service-bus-go"]
 
 # Kubernetes incubator deps
 [[constraint]]
-  revision = "1f1cda41a301080789507a291b999807002ede46" #revision for 1.11.2 release
   name = "github.com/kubernetes-incubator/custom-metrics-apiserver"
+  version = "kubernetes-1.11.2"
 
 [[constraint]]
   name = "github.com/Azure/azure-sdk-for-go"
   version = "21.0.0"
-
+
+# temp till Kubernetes incubator deps gets update 
+# https://github.com/kubernetes/kubernetes/pull/66078
+[[constraint]]
+  name = "k8s.io/client-go"
+  version = "kubernetes-1.11.4"
+
 [prune]
   go-tests = true
   unused-packages = true

README.md

@@ -95,9 +95,7 @@ Common external metrics to use for autoscaling are:
 
 ## Custom Metrics
 
-Custom metrics are currently retrieved from Application Insights.  Currently you will need to use an [AppId and API key](https://dev.applicationinsights.io/documentation/Authorization/API-key-and-App-ID) to authenticate and retrieve metrics from Application Insights.  View a list of basic metrics that come out of the box and see sample values at the [AI api explorer](https://dev.applicationinsights.io/apiexplorer/metrics).  
-
-> note: Metrics that have multi parts currently need will be passed as with a separator of `-` in place of AI separator of `/`.  An example is `performanceCounters/requestsPerSecond` should be specified as `performanceCounters-requestsPerSecond`
+Custom metrics are currently retrieved from Application Insights.  View a list of basic metrics that come out of the box and see sample values at the [AI api explorer](https://dev.applicationinsights.io/apiexplorer/metrics).  
 
 Common Custom Metrics are:
 

charts/azure-k8s-metrics-adapter/templates/cluster-role.yaml

@@ -67,6 +67,7 @@ rules:
   - azure.com
   resources:
   - "externalmetrics"
+  - "custommetrics"
   verbs:
   - list
   - get

charts/azure-k8s-metrics-adapter/templates/deployment.yaml

@@ -79,6 +79,18 @@ spec:
                   name: {{ template "azure-k8s-metrics-adapter.fullname" . }}
                   key: azure-subscription-id
           {{- end }}
+          {{- if .Values.appInsights.appId }}
+            - name: APP_INSIGHTS_APP_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "azure-k8s-metrics-adapter.fullname" . }}
+                  key: appinsights-appid
+            - name: APP_INSIGHTS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "azure-k8s-metrics-adapter.fullname" . }}
+                  key: appinsights-key
+          {{- end }}
           {{- range $key, $value := .Values.extraEnv }}
             - name: {{ $key }}
               value: "{{ $value }}"

charts/azure-k8s-metrics-adapter/templates/secret.yaml

@@ -24,4 +24,8 @@ data:
   {{- if .Values.defaultSubscriptionId }}
   azure-subscription-id: {{ .Values.defaultSubscriptionId | b64enc | quote }}
   {{- end }}
+  {{- if .Values.appInsights.appId }}
+  appinsights-appid: {{ .Values.appInsights.appId | b64enc | quote }}
+  appinsights-key: {{ .Values.appInsights.key | b64enc | quote }}
+  {{- end }}
 {{- end }}

charts/azure-k8s-metrics-adapter/values.yaml

@@ -55,6 +55,12 @@ azureAuthentication:
   azureIdentityClientId: ""
 
 
+# It is possible to pass app insights app id and key instead of using service principle
+# if no application insights key has been specified, then we will use AD authentication
+appInsights:
+  appId: ""
+  key: ""
+
 # Override the subscription if outside of Azure or for full control with `SUBSCRIPTION_ID`
 # See https://github.com/jsturtevant/azure-k8-metrics-adapter#subscription-information
 defaultSubscriptionId: ""

deploy/adapter.yaml

@@ -130,6 +130,7 @@ rules:
   - azure.com
   resources:
   - "externalmetrics"
+  - "custommetrics"
   verbs:
   - list
   - get

local-dev-values.yaml.example

@@ -6,4 +6,8 @@ azureAuthentication:
   clientID: ""
   clientSecret: ""
 
-defaultSubscriptionId: ""
+defaultSubscriptionId: ""
+
+appInsights:
+  appId: ""
+  key: ""

main.go

@@ -61,11 +61,16 @@ func setupAzureProvider(cmd *basecmd.AdapterBase, metricsCache *metriccache.Metr
 		glog.Fatalf("unable to construct discovery REST mapper: %v", err)
 	}
 
+	dynamicClient, err := cmd.DynamicClient()
+	if err != nil {
+		glog.Fatalf("unable to construct dynamic k8s client: %v", err)
+	}
+
 	defaultSubscriptionID := getDefaultSubscriptionID()
 	monitorClient := monitor.NewClient(defaultSubscriptionID)
 	appinsightsClient := appinsights.NewClient()
 
-	azureProvider := azureprovider.NewAzureProvider(defaultSubscriptionID, mapper, appinsightsClient, monitorClient, metricsCache)
+	azureProvider := azureprovider.NewAzureProvider(defaultSubscriptionID, mapper, dynamicClient, appinsightsClient, monitorClient, metricsCache)
 	cmd.WithCustomMetrics(azureProvider)
 	cmd.WithExternalMetrics(azureProvider)
 }
@@ -81,9 +86,12 @@ func newController(cmd *basecmd.AdapterBase, metricsCache *metriccache.MetricCac
 	}
 
 	adapterInformerFactory := informers.NewSharedInformerFactory(adapterClientSet, time.Second*30)
+	handler := controller.NewHandler(adapterInformerFactory.Azure().V1alpha1().ExternalMetrics().Lister(),
+		adapterInformerFactory.Azure().V1alpha1().CustomMetrics().Lister(),
+		metricsCache)
 
-	handler := controller.NewHandler(adapterInformerFactory.Azure().V1alpha1().ExternalMetrics().Lister(), metricsCache)
-	controller := controller.NewController(adapterInformerFactory.Azure().V1alpha1().ExternalMetrics(), &handler)
+	controller := controller.NewController(adapterInformerFactory.Azure().V1alpha1().ExternalMetrics(),
+		adapterInformerFactory.Azure().V1alpha1().CustomMetrics(), &handler)
 
 	return controller, adapterInformerFactory
 }

pkg/apis/metrics/v1alpha1/custommetric.go

@@ -28,9 +28,9 @@ type CustomMetricSpec struct {
 
 // CustomMetricConfig holds app insights configuration
 type CustomMetricConfig struct {
-	metricName    string `json:"metricName"`
-	applicationID string `json:"applicationID"`
-	query         string `json:"query"`
+	MetricName    string `json:"metricName"`
+	ApplicationID string `json:"applicationID"`
+	Query         string `json:"query"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/azure/appinsights/appinsights.go

@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/appinsights/v1/insights"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
@@ -26,7 +25,7 @@ const (
 
 // AzureAppInsightsClient provides methods for accessing App Insights via AD auth or App API Key
 type AzureAppInsightsClient interface {
-	GetCustomMetric(namespace string, metricName string) (float64, error)
+	GetCustomMetric(request MetricRequest) (float64, error)
 }
 
 // appinsightsClient is used to call Application Insights Api
@@ -51,20 +50,16 @@ func NewClient() AzureAppInsightsClient {
 }
 
 // GetCustomMetric calls to Application Insights to retrieve the value of the metric requested
-func (c appinsightsClient) GetCustomMetric(namespace string, metricName string) (float64, error) {
-	// because metrics names are multipart in AI and we can not pass an extra /
-	// through k8s api we convert - to / to get around that
-	convertedMetricName := strings.Replace(metricName, "-", "/", -1)
-	glog.V(2).Infof("New call to GetCustomMetric: %s", convertedMetricName)
+func (c appinsightsClient) GetCustomMetric(request MetricRequest) (float64, error) {
 
 	// get the last 5 mins and chunking into 30 seconds
 	// this seems to be the best way to get the closest average rate at time of request
 	// any smaller time intervals and the values come back null
-	metricRequestInfo := NewMetricRequest(convertedMetricName)
-	metricRequestInfo.Timespan = "PT5M"
-	metricRequestInfo.Interval = "PT30S"
+	// TODO make this configurable?
+	request.Timespan = "PT5M"
+	request.Interval = "PT30S"
 
-	metricsResult, err := c.getMetric(metricRequestInfo)
+	metricsResult, err := c.getMetric(request)
 	if err != nil {
 		return 0, err
 	}
@@ -80,7 +75,7 @@ func (c appinsightsClient) GetCustomMetric(namespace string, metricName string)
 	}
 
 	// grab just the last value which will be the latest value of the metric
-	metric := segments[len(segments)-1].AdditionalProperties[convertedMetricName]
+	metric := segments[len(segments)-1].AdditionalProperties[request.MetricName]
 	metricMap := metric.(map[string]interface{})
 	value := metricMap["avg"]
 	normalizedValue := normalizeValue(value)