开发Kubernetes Operator

Operator 介绍

Operator Framework

示例

apiVersion: app.example.com/v1
kind: AppService
metadata:
 name: nginx-app
spec:
 replicas: 2
 image: nginx:1.7.9
 ports:
 - port: 80
 targetPort: 80
 nodePort: 30002

开发环境

wget -c https://github.com/operator-framework/operator-sdk/releases/download/v1.17.0/operator-sdk_linux_amd64
mv operator-sdk_linux_amd64 /usr/bin/operator-sdk
chmod +x /usr/bin/operator-sdk

yum -y install gcc

Operator开发

# 创建项目目录
mkdir -p my-operator && cd my-operator
# 使用gomodules包管理工具
export GO111MODULE=on 
# 使用代理加速
export GOPROXY="https://goproxy.cn" 
# 使用 sdk 创建一个名为 my-operator 的 operator 项目
go mod init my-operator
# 使用下面的命令初始化项目
operator-sdk init --domain example.com --license apache2 --owner "developer"

$ tree -L 2
.
├── config
│  ├── default
│  ├── manager
│  ├── manifests
│  ├── prometheus
│  ├── rbac
│  └── scorecard
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│  └── boilerplate.go.txt
├── main.go
├── Makefile
└── PROJECT
8 directories, 7 files

项目结构

添加 API

$ operator-sdk create api --resource=true --controller=true --group app --version v1 --kind AppService
$ go mod tidy

自定义 API

type AppServiceSpec struct {
	Replicas *int32 `json:"replicas"`
	Image string `json:"image"`
	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
	Envs []corev1.EnvVar `json:"envs,omitempty"`
	Ports []corev1.ServicePort `json:"ports,omitempty"`
}

import (
 appsv1 "k8s.io/api/apps/v1"
 corev1 "k8s.io/api/core/v1"
 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type AppServiceStatus struct {
	appsv1.DeploymentStatus `json:",inline"`
}

$ make
/Users/ych/devs/projects/go/bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
go fmt ./...
go vet ./...
go build -o bin/manager main.go

实现业务逻辑

type AppServiceReconciler struct {
	client.Client
	Log logr.Logger
	Scheme *runtime.Scheme
}

var oldSpecAnnotation = "old/spec"

func (r *AppServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 ctx = context.Background()
 log := r.Log.WithValues("appservice", req.NamespacedName)
	// 获取appService crd资源
	appService := &appv1.AppService{}
	if err := r.Client.Get(ctx, req.NamespacedName, appService); err != nil {
		if errors.IsNotFound(err) {
			return ctrl.Result{}, nil
		}
		return ctrl.Result{}, err
	}
	// crd 资源标记为删除
	if appService.DeletionTimestamp != nil {
		return ctrl.Result{}, nil
	}
	log.Info("fetch appservice objects", "appservice", appService)
	// 如果不存在,则创建关联资源; 如果存在,判断是否需要更新
	// 如果需要更新,则直接更新; 如果不需要更新,则正常返回
	oldDeploy := &appsv1.Deployment{}
	if err := r.Client.Get(ctx, req.NamespacedName, oldDeploy); err != nil {
		// deployment 不存在,创建
		if errors.IsNotFound(err) {
			// 创建deployment
			if err := r.Client.Create(ctx, resources.NewDeploy(appService)); err != nil {
				return ctrl.Result{}, err
			}
			// 创建service
			if err := r.Client.Create(ctx, resources.NewService(appService)); err != nil {
				return ctrl.Result{}, err
			}
			// 更新 crd 资源的 Annotations
			data, _ := json.Marshal(appService.Spec)
			if appService.Annotations != nil {
				appService.Annotations["spec"] = string(data)
			} else {
				appService.Annotations = map[string]string{"spec": string(data)}
			}
			if err := r.Client.Update(ctx, appService); err != nil {
				return ctrl.Result{}, err
			}
		} else {
			return ctrl.Result{}, err
		}
	} else {
		// deployment 存在,更新
		oldSpec := appv1.AppServiceSpec{}
		if err := json.Unmarshal([]byte(appService.Annotations["spec"]), &oldSpec); err != nil {
			return ctrl.Result{}, err
		}
		if !reflect.DeepEqual(appService.Spec, oldSpec) {
			// 更新deployment
			newDeploy := resources.NewDeploy(appService)
			oldDeploy.Spec = newDeploy.Spec
			if err := r.Client.Update(ctx, oldDeploy); err != nil {
				return ctrl.Result{}, err
			}
			// 更新service
			newService := resources.NewService(appService)
			oldService := &corev1.Service{}
			if err := r.Client.Get(ctx, req.NamespacedName, oldService); err != nil {
				return ctrl.Result{}, err
			}
			// 更新 service 必须设置老的 clusterIP
			clusterIP := oldService.Spec.ClusterIP
			oldService.Spec = newService.Spec
			oldService.Spec.ClusterIP = clusterIP
			if err := r.Client.Update(ctx, oldService); err != nil {
				return ctrl.Result{}, err
			}
			// 更新 crd 资源的 Annotations
			data, _ := json.Marshal(appService.Spec)
			if appService.Annotations != nil {
				appService.Annotations["spec"] = string(data)
			} else {
				appService.Annotations = map[string]string{"spec": string(data)}
			}
			if err := r.Client.Update(ctx, appService); err != nil {
				return ctrl.Result{}, err
			}
		}
	}
	return ctrl.Result{}, nil
}

var (
	GroupVersion = schema.GroupVersion{Group: "app.example.com", Version: "v1"}
	Kind = "AppService"
 SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
 AddToScheme = SchemeBuilder.AddToScheme
)

/resources/deployment.go

/resources/service.go

调试

$ kubectl cluster-info

$ make install 
$ kubectl get crd |grep appservice
appservices.app.example.com 2022年02月27日T07:31:10Z

$ make run ENABLE_WEBHOOKS=false

apiVersion: app.ydzs.io/v1beta1
kind: AppService
metadata:
 name: nginx
spec:
 replicas: 2
 image: nginx:1.7.9
 ports:
 - port: 80
 targetPort: 80
 nodePort: 30002

$ kubectl apply -f config/samples/app_v1_appservice.yaml

......
1.646208501373661e+09	INFO	controllers.AppService	fetch appservice objects	{"appservice": "default/nginx", "appservice": {"apiVersion": "app.example.com/v1", "kind": "AppService", "namespace": "default", "name": "nginx"}}
1.6462085017729385e+09	INFO	controllers.AppService	fetch appservice objects	{"appservice": "default/nginx", "appservice": {"apiVersion": "app.example.com/v1", "kind": "AppService", "namespace": "default", "name": "nginx"}}
......

$ kubectl get AppService
NAME AGE
nginx 2m8s
$ kubectl get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
nginx p 2/2 2 2 2m20s
$ kubectl get svc 
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx NodePort 10.111.179.0 <none> 80:30002/TCP 2m23s

$ kubectl delete -f config/samples/app_v1_appservice.yaml
$ make uninstall

部署

# Build the manager binary
FROM golang:1.17 as builder
WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN export GOPROXY=https://goproxy.cn && go mod download
# Copy the go source
COPY main.go main.go
COPY api/ api/
COPY controllers/ controllers/
COPY resources/ resources/
# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM golang:1.17
WORKDIR /
COPY --from=builder /workspace/manager .
#USER 65532:65532
ENTRYPOINT ["/manager"]

$ make docker-build IMG=registry.cn-hangzhou.aliyuncs.com/test-operator/my-operator:v1.0.0

$ make docker-push IMG=registry.cn-hangzhou.aliyuncs.com/test-operator/my-operator:v1.0.0

apiVersion: v1
kind: Namespace
metadata:
 labels:
 control-plane: controller-manager
 name: system
---
apiVersion: apps/v1
kind: Deployment
metadata:
 name: controller-manager
 namespace: system
 labels:
 control-plane: controller-manager
spec:
 selector:
 matchLabels:
 control-plane: controller-manager
 replicas: 1
 template:
 metadata:
 annotations:
 kubectl.kubernetes.io/default-container: manager
 labels:
 control-plane: controller-manager
 spec:
 securityContext:
 runAsNonRoot: true
 containers:
 - command:
 - /manager
 args:
 - --leader-elect
 image: registry.cn-hangzhou.aliyuncs.com/test-operator/my-operator:v1.0.0
 name: manager
 securityContext:
 allowPrivilegeEscalation: false
 livenessProbe:
 httpGet:
 path: /healthz
 port: 8081
 initialDelaySeconds: 15
 periodSeconds: 20
 readinessProbe:
 httpGet:
 path: /readyz
 port: 8081
 initialDelaySeconds: 5
 periodSeconds: 10
 # TODO(user): Configure the resources accordingly based on the project requirements.
 # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 resources:
 limits:
 cpu: 500m
 memory: 128Mi
 requests:
 cpu: 10m
 memory: 64Mi
 serviceAccountName: controller-manager
 terminationGracePeriodSeconds: 10

containers:
- name: kube-rbac-proxy
 image: kubesphere/kube-rbac-proxy:v0.8.0

部署operator

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
 name: manager-rolebinding
roleRef:
 apiGroup: rbac.authorization.k8s.io
 kind: ClusterRole
 #name: manager-role
 name: cluster-admin
subjects:
 - kind: ServiceAccount
 name: controller-manager
 namespace: system

$ touch config/rbac/role.yaml # 创建一个空文件,否则部署会报错
$ make deploy IMG=registry.cn-hangzhou.aliyuncs.com/test-operator/my-operator:v1.0.0
/root/my-operator/bin/controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
cd config/manager && /root/my-operator/bin/kustomize edit set image controller=registry.cn-hangzhou.aliyuncs.com/test-operator/my-operator:v1.0.0
/root/my-operator/bin/kustomize build config/default | kubectl apply -f -
namespace/my-operator-system created
customresourcedefinition.apiextensions.k8s.io/appservices.app.example.com created
serviceaccount/my-operator-controller-manager created
role.rbac.authorization.k8s.io/my-operator-leader-election-role created
clusterrole.rbac.authorization.k8s.io/my-operator-metrics-reader created
clusterrole.rbac.authorization.k8s.io/my-operator-proxy-role created
rolebinding.rbac.authorization.k8s.io/my-operator-leader-election-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/my-operator-manager-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/my-operator-proxy-rolebinding created
configmap/my-operator-manager-config created
service/my-operator-controller-manager-metrics-service created
deployment.apps/my-operator-controller-manager created

# kubectl get po -A | grep my-operator
my-operator-system my-operator-controller-manager-b5f8bcb58-7nqkv 2/2 Running 0 11h
# kubectl get svc -A | grep my-operator
my-operator-system my-operator-controller-manager-metrics-service ClusterIP 172.21.2.239 <none> 8443/TCP 11h

$ kubectl apply -f config/samples/app_v1_appservice.yaml
$ kubectl get deploy |grep nginx
nginx 2/2 2 2 46s
$ kubectl get pod |grep nginx
nginx-5bf87f5f59-knbdw 1/1 Running 0 51s
nginx-5bf87f5f59-lblx6 1/1 Running 0 51s
$ kubectl get svc |grep nginx
nginx NodePort 172.21.0.149 <none> 80:30002/TCP 57s

$ kubectl delete -f config/samples/app_v1_appservice.yaml

$ make uninstall

$ make undeploy