鲜花的主人
文章
216
分类
23
评论
21

k8s部署MongoDB集群

| Kubernetes • MongoDB •
字数总计 11988 | 阅读时长 29分钟 | 阅读量 721

环境介绍

MongoDB介绍

MongoDB 是由C++语言编写的，是一个基于分布式文件存储的开源数据库系统，在高负载的情况下，添加更多的节点，可以保证服务器性能
MongoDB 旨在为WEB应用提供可扩展的高性能数据存储解决方案
MongoDB 将数据存储为一个文档，数据结构由键值(key=>value)对组成，文档类似于 JSON 对象，字段值可以包含其他文档，数组及文档数组
k8smo1.png

部署规划

本文使用StatefulSet部署MongoDB集群，同时每个MongoDB实例使用nfs实现持久化存储,从而部署无单点故障、高可用、可动态扩展的MongoDB集群
部署架构如下：
k8smo2.png

部署MongoDB集群

禁用巨页

运行一个 DaemonSet 的控制器来管理节点禁用巨页，因为 MongoDB 是建议关闭掉 Transparent Hugepage 的，否则可能导致性能下降，内存锁，甚至系统重启等问题，当然最好的还是只调整 MongoDB 的 Pod 所在的节点

[root@k8s01 mongodb]# cat hostvm-ds.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: hostvm-configurer
  labels:
    app: startup-script
spec:
  selector:
    matchLabels:
      app: startup-script
  template:
    metadata:
      labels:
        app: startup-script
    spec:
      hostPID: true
      containers:
      - name: hostvm-configurer
        image: cnych/startup-script:v1
        securityContext:
          privileged: true
        env:
        - name: STARTUP_SCRIPT
          value: |
            #! /bin/bash
            set -o errexit
            set -o pipefail
            set -o nounset
            
            # Disable hugepages
            echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled
            echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
[root@k8s01 mongodb]# kubectl apply -f hostvm-ds.yaml
daemonset.apps/hostvm-configurer created
[root@k8s01 mongodb]# kubectl get po -owide|grep hostvm
hostvm-configurer-8hb5j            1/1     Running   0          113s   10.244.2.54   k8s03   <none>           <none>
hostvm-configurer-r47fn            1/1     Running   0          113s   10.244.1.91   k8s02   <none>           <none>
hostvm-configurer-zbwlw            1/1     Running   0          113s   10.244.0.56   k8s01   <none>           <none>

部署mongodb集群

部署 ServiceAccount、Headless SVC 和 StatefulSet

[root@k8s01 mongodb]# cat mongo.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: mongo
  namespace: tool-env
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: mongo
subjects:
  - kind: ServiceAccount
    name: mongo
    namespace: tool-env
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: Service
metadata:
 name: mongo
 namespace: tool-env
 labels:
   name: mongo
spec:
 ports:
 - port: 27017
   targetPort: 27017
 clusterIP: None
 selector:
   role: mongo
---
apiVersion: v1
kind: Service
metadata:
 name: mongo-client
 namespace: tool-env
 labels:
   name: mongo
spec:
 ports:
 - port: 27017
   targetPort: 27017
   nodePort: 32717
 type: NodePort
 selector:
   role: mongo
---   
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mongo
  namespace: tool-env
spec:
  serviceName: mongo
  replicas: 3
  selector:
    matchLabels:
      role: mongo
      environment: staging
  template:
    metadata:
      labels:
        role: mongo
        environment: staging
        replicaset: MainRepSet
    spec:
      affinity:
        podAntiAffinity:  # 添加 Pod 反亲和性，将副本打散在不同的节点
          preferredDuringSchedulingIgnoredDuringExecution:  # 软策略
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: replicaset
                  operator: In
                  values:
                  - MainRepSet
              topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 10
      serviceAccountName: mongo
      containers:
        - name: mongo
          image: mongo:4.0
          command:
            - mongod
            - "--wiredTigerCacheSizeGB"
            - "0.25"
            - "--bind_ip"
            - "0.0.0.0"
            - "--replSet"
            - MainRepSet
            - "--smallfiles"
            - "--noprealloc"
          ports:
            - containerPort: 27017
          volumeMounts:
            - name: mongo-data
              mountPath: /data/db
          resources:
            requests:
              cpu: 1
              memory: 2Gi
        - name: mongo-sidecar
          image: cvallance/mongo-k8s-sidecar
          env:
            - name: MONGO_SIDECAR_POD_LABELS
              value: "role=mongo,environment=staging"
            - name: KUBE_NAMESPACE   #根据自己namespace
              value: "tool-env"
            - name: KUBERNETES_MONGO_SERVICE_NAME
              value: "mongo"
  volumeClaimTemplates:
  - metadata:
      name: mongo-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: managed-nfs-storage  # 提供一个可用的 Storageclass
      resources:
        requests:
          storage: 10Gi
[root@k8s01 mongodb]# kubectl apply -f mongo.yaml
serviceaccount/mongo created
clusterrolebinding.rbac.authorization.k8s.io/mongo created
service/mongo created
service/mongo-client created
statefulset.apps/mongo created

这里给 Mongo 的 Pod 添加了一个 sidecar 容器，主要用于副本集的配置，该 sidecar 会每5s检查一次新成员，通过几个环境变量配置指定了 Pod 的标签、命名空间和 Service，为了保证应用的稳定性，通过 podAntiAffinity 指定了 Pod 的反亲和性，这样可以保证不会有两个副本出现在同一个节点上
部署完成后可以通过如下命令检查应用运行状态

[root@k8s01 mongodb]# kubectl -n tool-env get all
NAME          READY   STATUS    RESTARTS   AGE
pod/mongo-0   2/2     Running   0          11m
pod/mongo-1   2/2     Running   0          9m35s
pod/mongo-2   2/2     Running   0          7m9s

NAME                   TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)          AGE
service/mongo          ClusterIP   None         <none>        27017/TCP        11m

NAME                     READY   AGE
statefulset.apps/mongo   3/3     11m

验证集群

[root@k8s01 mongodb]# kubectl exec -it -n tool-env mongo-0 bash -c mongo
root@mongo-0:/# mongo
MongoDB shell version v4.0.19
connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb
.......
MainRepSet:SECONDARY> rs.status
function () {
    return db._adminCommand("replSetGetStatus");
}
MainRepSet:SECONDARY> rs.status()
{
	"set" : "MainRepSet",
	"date" : ISODate("2020-08-08T14:32:31.132Z"),
	"myState" : 2,
	"term" : NumberLong(1),
	"syncingTo" : "mongo-1.mongo.tool-env.svc.cluster.local:27017",
	"syncSourceHost" : "mongo-1.mongo.tool-env.svc.cluster.local:27017",
	"syncSourceId" : 1,
	"heartbeatIntervalMillis" : NumberLong(2000),
	"optimes" : {
		"lastCommittedOpTime" : {
			"ts" : Timestamp(1596897144, 1),
			"t" : NumberLong(1)
		},
		"readConcernMajorityOpTime" : {
			"ts" : Timestamp(1596897144, 1),
			"t" : NumberLong(1)
		},
		"appliedOpTime" : {
			"ts" : Timestamp(1596897144, 1),
			"t" : NumberLong(1)
		},
		"durableOpTime" : {
			"ts" : Timestamp(1596897144, 1),
			"t" : NumberLong(1)
		}
	},
	"lastStableCheckpointTimestamp" : Timestamp(1596897114, 1),
	"members" : [
		{
			"_id" : 0,
			"name" : "mongo-2.mongo.tool-env.svc.cluster.local:27017",
			"health" : 1,
			"state" : 1,
			"stateStr" : "PRIMARY",
			"uptime" : 171,
			"optime" : {
				"ts" : Timestamp(1596897144, 1),
				"t" : NumberLong(1)
			},
			"optimeDurable" : {
				"ts" : Timestamp(1596897144, 1),
				"t" : NumberLong(1)
			},
			"optimeDate" : ISODate("2020-08-08T14:32:24Z"),
			"optimeDurableDate" : ISODate("2020-08-08T14:32:24Z"),
			"lastHeartbeat" : ISODate("2020-08-08T14:32:30.035Z"),
			"lastHeartbeatRecv" : ISODate("2020-08-08T14:32:29.871Z"),
			"pingMs" : NumberLong(0),
			"lastHeartbeatMessage" : "",
			"syncingTo" : "",
			"syncSourceHost" : "",
			"syncSourceId" : -1,
			"infoMessage" : "",
			"electionTime" : Timestamp(1596896964, 2),
			"electionDate" : ISODate("2020-08-08T14:29:24Z"),
			"configVersion" : 4
		},
		{
			"_id" : 1,
			"name" : "mongo-1.mongo.tool-env.svc.cluster.local:27017",
			"health" : 1,
			"state" : 2,
			"stateStr" : "SECONDARY",
			"uptime" : 171,
			"optime" : {
				"ts" : Timestamp(1596897144, 1),
				"t" : NumberLong(1)
			},
			"optimeDurable" : {
				"ts" : Timestamp(1596897144, 1),
				"t" : NumberLong(1)
			},
			"optimeDate" : ISODate("2020-08-08T14:32:24Z"),
			"optimeDurableDate" : ISODate("2020-08-08T14:32:24Z"),
			"lastHeartbeat" : ISODate("2020-08-08T14:32:30.034Z"),
			"lastHeartbeatRecv" : ISODate("2020-08-08T14:32:30.369Z"),
			"pingMs" : NumberLong(0),
			"lastHeartbeatMessage" : "",
			"syncingTo" : "mongo-2.mongo.tool-env.svc.cluster.local:27017",
			"syncSourceHost" : "mongo-2.mongo.tool-env.svc.cluster.local:27017",
			"syncSourceId" : 0,
			"infoMessage" : "",
			"configVersion" : 4
		},
		{
			"_id" : 2,
			"name" : "mongo-0.mongo.tool-env.svc.cluster.local:27017",
			"health" : 1,
			"state" : 2,
			"stateStr" : "SECONDARY",
			"uptime" : 196,
			"optime" : {
				"ts" : Timestamp(1596897144, 1),
				"t" : NumberLong(1)
			},
			"optimeDate" : ISODate("2020-08-08T14:32:24Z"),
			"syncingTo" : "mongo-1.mongo.tool-env.svc.cluster.local:27017",
			"syncSourceHost" : "mongo-1.mongo.tool-env.svc.cluster.local:27017",
			"syncSourceId" : 1,
			"infoMessage" : "",
			"configVersion" : 4,
			"self" : true,
			"lastHeartbeatMessage" : ""
		}
	],
	"ok" : 1,
	"operationTime" : Timestamp(1596897144, 1),
	"$clusterTime" : {
		"clusterTime" : Timestamp(1596897144, 1),
		"signature" : {
			"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
			"keyId" : NumberLong(0)
		}
	}
}

Mongo集群扩容

如需要对mongo扩容，只需要调整statefulset的replicas即可

[root@k8s01 mongodb]# kubectl scale statefulset mongo --replicas=4 -n test-env

Mongo使用/访问

mongo cluster访问默认连接为：

mongodb://mongo1,mongo2,mongo3:27017/dbname_?

在kubernetes中最常用的FQDN连接服务的连接为：

appName.$HeadlessServiceName.$Namespace.svc.cluster.local

因为我们采用statefulset部署的pod，所以命名均有规则，所以实际上如果连接4副本的mongodb cluster，上面的默认连接该为（默认为namespace之外）：

mongodb://mongo-0.mongo.tool-env.svc.cluster.local:27017,mongo-1.mongo.tool-env.svc.cluster.local:27017,mongo-2.mongo.tool-env.svc.cluster.local:27017,mongo-3.mongo.tool-env.svc.cluster.local:2701
文章作者： 鲜花的主人
本文链接：
版权声明： 本站所有文章除特别声明外，均采用 CC BY-NC-SA 4.0 许可协议。转载请注明来自 爱吃可爱多！
Kubernetes MongoDB 数据库 Kubernetes
喜欢就支持一下吧
打赏
微信 微信
支付宝 支付宝