453 lines
14 KiB
Markdown
453 lines
14 KiB
Markdown
<h1><center>Kubernetes使用StorageClass动态供应PV</center></h1>
|
||
|
||
作者:行癫(盗版必究)
|
||
|
||
------
|
||
|
||
|
||
|
||
## 一:安装 NFS 插件
|
||
|
||
- GitHub地址:https://github.com/kubernetes-incubator/external-storage/tree/master/nfs/deploy/kubernetes
|
||
|
||

|
||
|
||
#### 1、配置名称空间
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# ls
|
||
01-namespace.yaml 03-nfs-provisioner.yaml 05-test-claim.yaml 07-nginx-statefulset.yaml
|
||
02-rbac.yaml 04-nfs-StorageClass.yaml 06-test-pod.yaml
|
||
```
|
||
|
||
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat<<EOF>>01-namespace.yaml
|
||
apiVersion: v1
|
||
kind: Namespace
|
||
metadata:
|
||
name: dev
|
||
EOF
|
||
```
|
||
|
||
- 确认信息
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get ns
|
||
NAME STATUS AGE
|
||
default Active 12d
|
||
dev Active 60m
|
||
kube-node-lease Active 12d
|
||
kube-public Active 12d
|
||
kube-system Active 12d
|
||
kubernetes-dashboard Active 12d
|
||
```
|
||
|
||
#### 2、配置授权
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat<<EOF>>02-rbac.yaml
|
||
apiVersion: v1
|
||
kind: ServiceAccount
|
||
metadata:
|
||
name: nfs-client-provisioner
|
||
# replace with namespace where provisioner is deployed
|
||
namespace: dev
|
||
---
|
||
kind: ClusterRole
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
metadata:
|
||
name: nfs-client-provisioner-runner
|
||
rules:
|
||
- apiGroups: [""]
|
||
resources: ["persistentvolumes"]
|
||
verbs: ["get", "list", "watch", "create", "delete"]
|
||
- apiGroups: [""]
|
||
resources: ["persistentvolumeclaims"]
|
||
verbs: ["get", "list", "watch", "update"]
|
||
- apiGroups: ["storage.k8s.io"]
|
||
resources: ["storageclasses"]
|
||
verbs: ["get", "list", "watch"]
|
||
- apiGroups: [""]
|
||
resources: ["events"]
|
||
verbs: ["create", "update", "patch"]
|
||
---
|
||
kind: ClusterRoleBinding
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
metadata:
|
||
name: managed-run-nfs-client-provisioner
|
||
subjects:
|
||
- kind: ServiceAccount
|
||
name: nfs-client-provisioner
|
||
# replace with namespace where provisioner is deployed
|
||
namespace: dev
|
||
roleRef:
|
||
kind: ClusterRole
|
||
name: nfs-client-provisioner-runner
|
||
apiGroup: rbac.authorization.k8s.io
|
||
---
|
||
kind: Role
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
metadata:
|
||
name: leader-locking-nfs-client-provisioner
|
||
# replace with namespace where provisioner is deployed
|
||
namespace: dev
|
||
rules:
|
||
- apiGroups: [""]
|
||
resources: ["endpoints"]
|
||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||
---
|
||
kind: RoleBinding
|
||
apiVersion: rbac.authorization.k8s.io/v1
|
||
metadata:
|
||
name: leader-locking-nfs-client-provisioner
|
||
namespace: dev
|
||
subjects:
|
||
- kind: ServiceAccount
|
||
name: nfs-client-provisioner
|
||
# replace with namespace where provisioner is deployed
|
||
namespace: dev
|
||
roleRef:
|
||
kind: Role
|
||
name: leader-locking-nfs-client-provisioner
|
||
apiGroup: rbac.authorization.k8s.io
|
||
EOF
|
||
```
|
||
|
||
- 部署 rbac.yaml
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl create -f rbac.yaml
|
||
```
|
||
|
||
- 确认配置
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev clusterrole |grep nfs
|
||
nfs-client-provisioner-runner 2023-12-13T06:11:25Z
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev clusterrolebindings.rbac.authorization.k8s.io |grep nfs
|
||
managed-run-nfs-client-provisioner ClusterRole/nfs-client-provisioner-runner 57m
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev role
|
||
NAME CREATED AT
|
||
leader-locking-nfs-client-provisioner 2023-12-13T06:06:20Z
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev rolebindings.rbac.authorization.k8s.io
|
||
NAME ROLE AGE
|
||
leader-locking-nfs-client-provisioner Role/leader-locking-nfs-client-provisioner 58m
|
||
```
|
||
|
||
#### 3、创建nfs provisioner
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat<<EOF>>03-nfs-provisioner.yaml
|
||
apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: nfs-client-provisioner
|
||
labels:
|
||
app: nfs-client-provisioner
|
||
# replace with namespace where provisioner is deployed
|
||
namespace: dev #与RBAC文件中的namespace保持一致
|
||
spec:
|
||
replicas: 1
|
||
selector:
|
||
matchLabels:
|
||
app: nfs-client-provisioner
|
||
strategy:
|
||
type: Recreate
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: nfs-client-provisioner
|
||
spec:
|
||
serviceAccountName: nfs-client-provisioner
|
||
containers:
|
||
- name: nfs-client-provisioner
|
||
#image: quay.io/external_storage/nfs-client-provisioner:latest
|
||
image: easzlab/nfs-subdir-external-provisioner:v4.0.1
|
||
volumeMounts:
|
||
- name: nfs-client-root
|
||
mountPath: /persistentvolumes
|
||
env:
|
||
- name: PROVISIONER_NAME
|
||
value: provisioner-nfs-storage #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
|
||
- name: NFS_SERVER
|
||
value: master01 #NFS Server IP地址
|
||
- name: NFS_PATH
|
||
value: /data/volumes/v1 #NFS挂载卷
|
||
volumes:
|
||
- name: nfs-client-root
|
||
nfs:
|
||
server: master01 #NFS Server IP地址
|
||
path: /data/volumes/v1 #NFS 挂载卷
|
||
EOF
|
||
```
|
||
|
||
- 部署deployment-nfs.yaml
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl apply -f deployment-nfs.yaml
|
||
```
|
||
|
||
- 查看创建的POD
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get pods -n dev
|
||
NAME READY STATUS RESTARTS AGE
|
||
nfs-client-provisioner-6df46fcf96-qnh64 1/1 Running 0 53m
|
||
```
|
||
|
||
#### 4、创建-StorageClass
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat <<EOF>>04-nfs-StorageClass.yaml
|
||
apiVersion: storage.k8s.io/v1
|
||
kind: StorageClass
|
||
metadata:
|
||
name: managed-nfs-storage
|
||
namespace: dev
|
||
provisioner: provisioner-nfs-storage #这里的名称要和provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
|
||
parameters:
|
||
archiveOnDelete: "false"
|
||
EOF
|
||
```
|
||
|
||
- 部署storageclass-nfs.yaml
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl apply -f storageclass-nfs.yaml
|
||
```
|
||
|
||
- 查看创建的StorageClass
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get storageclasses.storage.k8s.io -n dev
|
||
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
|
||
managed-nfs-storage provisioner-nfs-storage Delete Immediate false 3h1m
|
||
```
|
||
|
||
#### 5、创建测试pvc
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat <<EOF>>05-test-claim.yaml
|
||
kind: PersistentVolumeClaim
|
||
apiVersion: v1
|
||
metadata:
|
||
name: test-claim
|
||
namespace: dev
|
||
## annotations:
|
||
## volume.beta.kubernetes.io/storage-class: "managed-nfs-storage" # 后期k8s不再支持这个注解,通过声明storageClassName的方式代替
|
||
spec:
|
||
accessModes:
|
||
- ReadWriteMany
|
||
storageClassName: managed-nfs-storage #与nfs-StorageClass.yaml metadata.name保持一致
|
||
resources:
|
||
requests:
|
||
storage: 5Mi
|
||
EOF
|
||
```
|
||
|
||
- 创建测试 PVC
|
||
|
||
```bash
|
||
[root@k8s-master StatefulSet]# kubectl create -f test-claim.yaml
|
||
```
|
||
|
||
- 查看创建的PVC状态为Bound
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev pvc
|
||
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
|
||
test-claim Bound pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3 1Mi RWX managed-nfs-storage 34m
|
||
```
|
||
|
||
- 查看自动创建的PV
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev pv
|
||
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
|
||
pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3 1Mi RWX Delete Bound default/test-claim managed-nfs-storage 34m
|
||
```
|
||
|
||
- 进入到NFS的export目录查看对应volume name的目录已经创建出来。
|
||
|
||
- 其中volume的名字是namespace,PVC name以及uuid的组合:
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# ls /data/volumes/v1
|
||
total 0
|
||
drwxrwxrwx 2 root root 21 Jan 29 12:03 default-test-claim-pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3
|
||
```
|
||
|
||
#### 6、创建测试pod
|
||
|
||
- 指定pod使用刚创建的PVC:test-claim,
|
||
|
||
- 完成之后,如果attach到pod中执行一些文件的读写操作,就可以确定pod的/mnt已经使用了NFS的存储服务了。
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cat<<EOF>>06-test-pod.yaml
|
||
kind: Pod
|
||
apiVersion: v1
|
||
metadata:
|
||
name: test-pod
|
||
namespace: dev
|
||
spec:
|
||
containers:
|
||
- name: test-pod
|
||
image: busybox:1.24
|
||
command:
|
||
- "/bin/sh"
|
||
args:
|
||
- "-c"
|
||
- "touch /mnt/SUCCESS && exit 0 || exit 1" #创建一个SUCCESS文件后退出
|
||
volumeMounts:
|
||
- name: nfs-pvc
|
||
mountPath: "/mnt"
|
||
restartPolicy: "Never"
|
||
volumes:
|
||
- name: nfs-pvc
|
||
persistentVolumeClaim:
|
||
claimName: test-claim #与PVC名称保持一致
|
||
EOF
|
||
```
|
||
|
||
- 执行yaml文件
|
||
|
||
```bash
|
||
[root@k8s-master StatefulSet]# kubectl apply -f test-pod.yaml
|
||
```
|
||
|
||
- 查看创建的测试POD
|
||
|
||
```swift
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev pod -o wide
|
||
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
|
||
nfs-client-provisioner-75bf876d88-578lg 1/1 Running 0 51m 10.244.2.131 k8s-node2 <none> <none>
|
||
test-pod 0/1 Completed 0 41m 10.244.1.
|
||
```
|
||
|
||
- 在NFS服务器上的共享目录下的卷子目录中检查创建的NFS PV卷下是否有"SUCCESS" 文件。
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# cd /data/volumes/v1
|
||
[root@k8s-master v1]# ll
|
||
total 0
|
||
drwxrwxrwx 2 root root 21 Jan 29 12:03 default-test-claim-pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3
|
||
|
||
[root@k8s-master v1]# cd default-test-claim-pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3/
|
||
[root@k8s-master default-test-claim-pvc-a17d9fd5-237a-11e9-a2b5-000c291c25f3]# ll
|
||
total 0
|
||
-rw-r--r-- 1 root root 0 Jan 29 12:03 SUCCESS
|
||
```
|
||
|
||
#### 7、清理测试环境
|
||
|
||
- 删除测试POD
|
||
|
||
```bash
|
||
[root@k8s-master StatefulSet]# kubectl delete -f test-pod.yaml
|
||
```
|
||
|
||
- 删除测试PVC
|
||
|
||
```bash
|
||
[root@k8s-master StatefulSet]# kubectl delete -f test-claim.yaml
|
||
```
|
||
|
||
- 在NFS服务器上的共享目录下查看NFS的PV卷已经被删除。
|
||
|
||
#### 8、创建一个nginx动态获取PV
|
||
|
||
|
||
```yaml
|
||
[root@k8s-master StatefulSet]# cat <<EOF>>nginx-statefulset.yaml
|
||
apiVersion: v1
|
||
kind: Service
|
||
metadata:
|
||
name: nginx-headless
|
||
namespace: dev
|
||
labels:
|
||
app: nginx
|
||
spec:
|
||
ports:
|
||
- port: 80
|
||
name: web
|
||
clusterIP: None #注意此处的值,None表示无头服务
|
||
selector:
|
||
app: nginx
|
||
---
|
||
apiVersion: apps/v1
|
||
kind: StatefulSet
|
||
metadata:
|
||
name: web
|
||
namespace: dev
|
||
spec:
|
||
selector:
|
||
matchLabels:
|
||
app: nginx
|
||
serviceName: "nginx-headless"
|
||
replicas: 3 #两个副本
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: nginx
|
||
spec:
|
||
containers:
|
||
- name: nginx
|
||
image: nginx
|
||
ports:
|
||
- containerPort: 80
|
||
name: web
|
||
volumeMounts:
|
||
- name: www
|
||
mountPath: /usr/share/nginx/html
|
||
volumeClaimTemplates: # 创建 pvc 模板
|
||
- metadata:
|
||
name: www
|
||
spec:
|
||
accessModes: [ "ReadWriteOnce" ]
|
||
storageClassName: "managed-nfs-storage"
|
||
resources:
|
||
requests:
|
||
storage: 1Gi
|
||
EOF
|
||
```
|
||
|
||
- 启动后看到以下信息:
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# kubectl get -n dev pods,pv,pvc
|
||
NAME READY STATUS RESTARTS AGE
|
||
pod/nfs-client-provisioner-5778d56949-ltjbt 1/1 Running 0 42m
|
||
pod/test-pod 0/1 Completed 0 36m
|
||
pod/web-0 1/1 Running 0 2m23s
|
||
pod/web-1 1/1 Running 0 2m6s
|
||
pod/web-2 1/1 Running 0 104s
|
||
|
||
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
|
||
persistentvolume/pvc-1d54bb5b-9c12-41d5-9295-3d827a20bfa2 1Gi RWO Delete Bound default/www-web-2 managed-nfs-storage 104s
|
||
persistentvolume/pvc-8cc0ed15-1458-4384-8792-5d4fd65dca66 1Gi RWO Delete Bound default/www-web-0 managed-nfs-storage 39m
|
||
persistentvolume/pvc-c924a2aa-f844-4d52-96c9-32769eb3f96f 1Mi RWX Delete Bound default/test-claim managed-nfs-storage 38m
|
||
persistentvolume/pvc-e30333d7-4aed-4700-b381-91a5555ed59f 1Gi RWO Delete Bound default/www-web-1 managed-nfs-storage 2m6s
|
||
|
||
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
|
||
persistentvolumeclaim/test-claim Bound pvc-c924a2aa-f844-4d52-96c9-32769eb3f96f 1Mi RWX managed-nfs-storage 38m
|
||
persistentvolumeclaim/www-web-0 Bound pvc-8cc0ed15-1458-4384-8792-5d4fd65dca66 1Gi RWO managed-nfs-storage 109m
|
||
persistentvolumeclaim/www-web-1 Bound pvc-e30333d7-4aed-4700-b381-91a5555ed59f 1Gi RWO managed-nfs-storage 2m6s
|
||
persistentvolumeclaim/www-web-2 Bound pvc-1d54bb5b-9c12-41d5-9295-3d827a20bfa2 1Gi RWO managed-nfs-storage 104s
|
||
```
|
||
|
||
- nfs服务器上也会看到自动生成3个挂载目录,当pod删除了数据还会存在。
|
||
|
||
```shell
|
||
[root@k8s-master StatefulSet]# ll /data/volumes/v1/
|
||
total 4
|
||
drwxrwxrwx 2 root root 6 Aug 16 18:21 default-nginx-web-0-pvc-ea22de1c-fa33-4f82-802c-f04fe3630007
|
||
drwxrwxrwx 2 root root 21 Aug 16 18:25 default-test-claim-pvc-c924a2aa-f844-4d52-96c9-32769eb3f96f
|
||
drwxrwxrwx 2 root root 6 Aug 16 18:21 default-www-web-0-pvc-8cc0ed15-1458-4384-8792-5d4fd65dca66
|
||
drwxrwxrwx 2 root root 6 Aug 16 18:59 default-www-web-1-pvc-e30333d7-4aed-4700-b381-91a5555ed59f
|
||
drwxrwxrwx 2 root root 6 Aug 16 18:59 default-www-web-2-pvc-1d54bb5b-9c12-41d5-9295-3d827a20bfa2
|
||
```
|
||
|