二进制部署Kubernetes集群1.33版本

1
# 解压到/usr/local/bin/ 目录下
2
kevn@k8s-master-01:~$ sudo tar -xvf containerd-2.1.0-linux-amd64.tar.gz -C /usr/local/
3

4
# 创建启动服务文件
5
koevn@k8s-master-01:~$ sudo cat > /etc/systemd/system/containerd.service <<EOF
6
[Unit]
7
Description=containerd container runtime
8
Documentation=https://containerd.io
9
After=network.target
10

11
[Service]
12
ExecStart=/usr/local/bin/containerd --config /etc/containerd/config.toml
13
Delegate=yes
14
KillMode=process
15
Restart=always
16
LimitNOFILE=1048576
17
LimitNPROC=infinity
18
LimitCORE=infinity
19

20
[Install]
21
WantedBy=multi-user.target
22
EOF
23

24
# 创建containerd 配置文件
25
koevn@k8s-master-01:~$ sudo mkdir -pv /etc/containerd/certs.d/harbor.koevn.com
26
koevn@k8s-master-01:~$ sudo containerd config default | tee /etc/containerd/config.toml
27
touch /etc/containerd/certs.d/harbor.koevn.com/hosts.toml
28

29
# 修改/etc/containerd/config.toml
30
[plugins]
31
  [plugins.'io.containerd.cri.v1.images']
32
    snapshotter = 'overlayfs'
33
    disable_snapshot_annotations = true
34
    discard_unpacked_layers = false
35
    max_concurrent_downloads = 3
36
    image_pull_progress_timeout = '5m0s'
37
    image_pull_with_sync_fs = false
38
    stats_collect_period = 10
39

40
    [plugins.'io.containerd.cri.v1.images'.pinned_images]
41
      sandbox = 'harbor.koevn.com/k8s/pause:3.10'    # 这个镜像拉起根据自己定义拉取，我这里是指向私有仓库
42

43
    [plugins.'io.containerd.cri.v1.images'.registry]
44
      config_path = "/etc/containerd/certs.d"  # 这里配置镜像仓库相关信息
45
    [plugins.'io.containerd.cri.v1.images'.image_decryption]
46
      key_model = 'node'
47

48
# 配置/etc/containerd/certs.d/harbor.koevn.com/hosts.toml
49
server = "https://harbor.koevn.com"
50

51
[host."https://harbor.koevn.com"]
52
  capabilities = ["pull", "resolve", "push"]
53
  skip_verify = true
54
  username = "admin"
55
  password = "K123456"
56

57
# 重新加载systemd管理
58
koevn@k8s-master-01:~$ sudo systemctl daemon-reload
59

60
# containerd服务启动
61
koevn@k8s-master-01:~$ sudo systemctl start containerd.service
62

63
# containerd服务状态
64
koevn@k8s-master-01:~$ sudo systemctl status containerd.service
65

66
# containerd服务开机自启
67
koevn@k8s-master-01:~$ sudo systemctl enable --now containerd.service

1
# 解压
2
koevn@k8s-master-01:~$ sudo tar xf crictl-v*-linux-amd64.tar.gz -C /usr/sbin/
3

4
# 生成配置文件
5
koevn@k8s-master-01:~$ sudo cat > /etc/crictl.yaml <<EOF
6
runtime-endpoint: unix:///run/containerd/containerd.sock
7
image-endpoint: unix:///run/containerd/containerd.sock
8
EOF
9

10
# 查看
11
crictl info

1
# 将已下载的cfssl工具移到/usr/local/bin目录下
2
koevn@k8s-master-01:/tmp$ sudo mv cfssl_1.6.5_linux_amd64 /usr/local/bin/cfssl
3
koevn@k8s-master-01:/tmp$ sudo mv cfssljson_1.6.5_linux_amd64 /usr/local/bin/cfssljson
4
koevn@k8s-master-01:/tmp$ sudo mv cfssl-certinfo_1.6.5_linux_amd64 /usr/local/bin/cfssl-certinfo
5
koevn@k8s-master-01:/tmp$ sudo chmod +x /usr/local/bin/cfss*
6

7
# 创建etcd服务目录
8
koevn@k8s-master-01:~$ sudo mkdir -pv /opt/etcd/{bin,cert,config}
9

10
# 解压etcd安装文件
11
koevn@k8s-master-01:/tmp$ sudo tar -xvf etcd*.tar.gz && mv etcd-*/etcd /opt/etcd/bin/
12

13
# 添加etcd系统环境变量
14
koevn@k8s-master-01:/tmp$ sudo echo "export PATH=/opt/etcd/bin:$PATH" > /etc/profile.d/etcd.sh && sudo source /etc/profile
15

16
# 创建证书生成目录
17
koevn@k8s-master-01:/tmp$ sudo mkdir -pv /opt/cert/etcd

1
koevn@k8s-master-01:/tmp$ cd /opt/cert
2
koevn@k8s-master-01:/opt/cert$ sudo cat > ca-config.json << EOF
3
{
4
  "signing": {
5
    "default": {
6
      "expiry": "876000h"
7
    },
8
    "profiles": {
9
      "kubernetes": {
10
        "usages": [
11
            "signing",
12
            "key encipherment",
13
            "server auth",
14
            "client auth"
15
        ],
16
        "expiry": "876000h"
17
      }
18
    }
19
  }
20
}
21
EOF

1
koevn@k8s-master-01:/opt/cert$ sudo cat > etcd-ca-csr.json  << EOF
2
{
3
  "CN": "etcd",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "etcd",
14
      "OU": "Etcd Security"
15
    }
16
  ],
17
  "ca": {
18
    "expiry": "876000h"
19
  }
20
}
21
EOF

1
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /opt/cert/etcd

1
koevn@k8s-master-01:/opt/cert$ sudo cat > etcd-csr.json << EOF
2
{
3
  "CN": "etcd",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "etcd",
14
      "OU": "Etcd Security"
15
    }
16
  ]
17
}
18
EOF

1
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert \
2
   -ca=/opt/cert/etcd/etcd-ca.pem \
3
   -ca-key=/opt/cert/etcd/etcd-ca-key.pem \
4
   -config=ca-config.json \
5
   -hostname=127.0.0.1,k8s-master-01,k8s-master-02,k8s-master-03,10.88.12.60,10.88.12.61,10.88.12.62 \
6
   -profile=kubernetes \
7
   etcd-csr.json | cfssljson -bare /opt/cert/etcd

1
koevn@k8s-master-01:~$ sudo cat > /oet/etcd/config/etcd.config.yml << EOF
2
name: 'k8s-master-01'   # 注意修改每个节点名称
3
data-dir: /data/etcd/data
4
wal-dir: /data/etcd/wal
5
snapshot-count: 5000
6
heartbeat-interval: 100
7
election-timeout: 1000
8
quota-backend-bytes: 0
9
listen-peer-urls: 'https://10.88.12.60:2380'  # 修改每个节点监听的IP地址
10
listen-client-urls: 'https://10.88.12.60:2379,http://127.0.0.1:2379'
11
max-snapshots: 3
12
max-wals: 5
13
cors:
14
initial-advertise-peer-urls: 'https://10.88.12.60:2380'
15
advertise-client-urls: 'https://10.88.12.60:2379'
16
discovery:
17
discovery-fallback: 'proxy'
18
discovery-proxy:
19
discovery-srv:
20
initial-cluster: 'k8s-master-01=https://10.88.12.60:2380,k8s-master-02=https://10.88.12.61:2380,k8s-master-03=https://10.88.12.62:2380'
21
initial-cluster-token: 'etcd-k8s-cluster'
22
initial-cluster-state: 'new'
23
strict-reconfig-check: false
24
enable-v2: true
25
enable-pprof: true
26
proxy: 'off'
27
proxy-failure-wait: 5000
28
proxy-refresh-interval: 30000
29
proxy-dial-timeout: 1000
30
proxy-write-timeout: 5000
31
proxy-read-timeout: 0
32
client-transport-security:
33
  cert-file: '/opt/etcd/cert/etcd.pem'
34
  key-file: '/opt/etcd/cert/etcd-key.pem'
35
  client-cert-auth: true
36
  trusted-ca-file: '/opt/etcd/cert/etcd-ca.pem'
37
  auto-tls: true
38
peer-transport-security:
39
  cert-file: '/opt/etcd/cert/etcd.pem'
40
  key-file: '/opt/etcd/cert/etcd-key.pem'
41
  peer-client-cert-auth: true
42
  trusted-ca-file: '/opt/etcd/cert/etcd-ca.pem'
43
  auto-tls: true
44
debug: false
45
log-package-levels:
46
log-outputs: [default]
47
force-new-cluster: false
48
EOF

1
koevn@k8s-master-01:~$ sudo tree /opt/etcd/
2
/opt/etcd/
3
├── bin
4
│   ├── etcd
5
│   ├── etcdctl
6
│   └── etcdutl
7
├── cert
8
│   ├── etcd-ca.pem
9
│   ├── etcd-key.pem
10
│   └── etcd.pem
11
└── config
12
    └── etcd.config.yml
13

14
4 directories, 7 files

1
koevn@k8s-master-01:~$ sudo tar -czvf /tmp/etcd.tar.gz /opt/etcd

1
koevn@k8s-master-01:~$ sudo cat /etc/systemd/system/etcd.service << EOF
2
[Unit]
3
Description=Etcd Service
4
Documentation=https://coreos.com/etcd/docs/latest/
5
After=network.target
6

7
[Service]
8
Type=notify
9
ExecStart=/opt/etcd/bin/etcd --config-file=/opt/etcd/config/etcd.config.yml
10
Restart=on-failure
11
RestartSec=10
12
LimitNOFILE=65536
13

14
[Install]
15
WantedBy=multi-user.target
16
Alias=etcd3.service
17
EOF

1
koevn@k8s-master-01:~$ sudo export ETCDCTL_API=3
2
root@k8s-master-01:~# etcdctl --endpoints="10.88.12.60:2379,10.88.12.61:2379,10.88.12.62:2379" \
3
> --cacert=/opt/etcd/cert/etcd-ca.pem \
4
> --cert=/opt/etcd/cert/etcd.pem \
5
> --key=/opt/etcd/cert/etcd-key.pem \
6
> endpoint status \
7
> --write-out=table
8
+------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
9
|     ENDPOINT     |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
10
+------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
11
| 10.88.12.60:2379 | c1f013862d84cb12 |  3.5.21 |   35 MB |     false |      false |       158 |    8434769 |            8434769 |        |
12
| 10.88.12.61:2379 | cea2c8779e4914d0 |  3.5.21 |   36 MB |     false |      false |       158 |    8434769 |            8434769 |        |
13
| 10.88.12.62:2379 | df3b2276b87896db |  3.5.21 |   33 MB |      true |      false |       158 |    8434769 |            8434769 |        |
14
+------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

1
koevn@k8s-master-01:~$ sudo mkdir -pv /opt/cert/kubernetes/pki
2
koevn@k8s-master-01:~$ sudo mkdir -pv /opt/kubernetes/cfg   # 集群kubeconfig文件目录
3
koevn@k8s-master-01:~$ cd /opt/cert

1
koevn@k8s-master-01:/opt/cert$ sudo cat > ca-csr.json << EOF
2
{
3
  "CN": "kubernetes",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "Kubernetes",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ],
17
  "ca": {
18
    "expiry": "876000h"
19
  }
20
}
21
EOF

1
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert -initca ca-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/ca

1
koevn@k8s-master-01:/opt/cert$ sudo cat > apiserver-csr.json << EOF
2
{
3
  "CN": "kube-apiserver",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "Kubernetes",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ]
17
}
18
EOF
19

20
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert   \
21
-ca=/opt/cert/kubernetes/pki/ca.pem   \
22
-ca-key=/opt/cert/kubernetes/pki/ca-key.pem   \
23
-config=ca-config.json   \
24
-hostname=10.90.0.1,10.88.12.100,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,k8s.koevn.com,10.88.12.60,10.88.12.61,10.88.12.62,10.88.12.63,10.88.12.64,10.88.12.65,10.88.12.66,10.88.12.67,10.88.12.68   \
25
-profile=kubernetes apiserver-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/apiserver

1
koevn@k8s-master-01:/opt/cert$ sudo cat > front-proxy-ca-csr.json << EOF
2
{
3
  "CN": "kubernetes",
4
  "key": {
5
     "algo": "rsa",
6
     "size": 2048
7
  },
8
  "ca": {
9
    "expiry": "876000h"
10
  }
11
}
12
EOF
13

14
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert \
15
-initca front-proxy-ca-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/front-proxy-ca
16

17
koevn@k8s-master-01:/opt/cert$ sudo cat > front-proxy-client-csr.json  << EOF
18
{
19
  "CN": "front-proxy-client",
20
  "key": {
21
     "algo": "rsa",
22
     "size": 2048
23
  }
24
}
25
EOF
26

27
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert  \
28
-ca=/opt/cert/kubernetes/pki/front-proxy-ca.pem   \
29
-ca-key=/opt/cert/kubernetes/pki/front-proxy-ca-key.pem   \
30
-config=ca-config.json   \
31
-profile=kubernetes \
32
front-proxy-client-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/front-proxy-client

1
koevn@k8s-master-01:/opt/cert$ sudo cat > manager-csr.json << EOF
2
{
3
  "CN": "system:kube-controller-manager",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "system:kube-controller-manager",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ]
17
}
18
EOF
19

20
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert  \
21
-ca=/opt/cert/kubernetes/pki/front-proxy-ca.pem   \
22
-ca-key=/opt/cert/kubernetes/pki/front-proxy-ca-key.pem   \
23
-config=ca-config.json   \
24
-profile=kubernetes \
25
manager-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/controller-manager
26

27
设置集群项
28
```bash
29
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-cluster kubernetes \
30
  --certificate-authority=/opt/cert/kubernetes/pki/ca.pem \
31
  --embed-certs=true \
32
  --server=https://10.88.12.100:9443 \
33
  --kubeconfig=/opt/kubernetes/cfg/controller-manager.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-context system:kube-controller-manager@kubernetes \
2
  --cluster=kubernetes \
3
  --user=system:kube-controller-manager \
4
  --kubeconfig=/opt/kubernetes/cfg/controller-manager.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-credentials system:kube-controller-manager \
2
  --client-certificate=/opt/cert/kubernetes/pki/controller-manager.pem \
3
  --client-key=/opt/cert/kubernetes/pki/controller-manager-key.pem \
4
  --embed-certs=true \
5
  --kubeconfig=/opt/kubernetes/cfg/controller-manager.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo kubectl config use-context system:kube-controller-manager@kubernetes \
2
  --kubeconfig=/opt/kubernetes/cfg/controller-manager.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo cat > scheduler-csr.json << EOF
2
{
3
  "CN": "system:kube-scheduler",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "system:kube-scheduler",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ]
17
}
18
EOF
19

20
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert \
21
  -ca=/opt/cert/kubernetes/pki/ca.pem \
22
  -ca-key=/opt/cert/kubernetes/pki/ca-key.pem \
23
  -config=ca-config.json \
24
  -profile=kubernetes \
25
  scheduler-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/scheduler
26

27
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-cluster kubernetes \
28
  --certificate-authority=/opt/cert/kubernetes/pki/ca.pem \
29
  --embed-certs=true \
30
  --server=https://10.88.12.100:9443 \
31
  --kubeconfig=/opt/kubernetes/cfg/scheduler.kubeconfig
32

33
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-credentials system:kube-scheduler \
34
  --client-certificate=/opt/cert/kubernetes/pki/scheduler.pem \
35
  --client-key=/opt/cert/kubernetes/pki/scheduler-key.pem \
36
  --embed-certs=true \
37
  --kubeconfig=/opt/kubernetes/cfg/scheduler.kubeconfig
38

39
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-context system:kube-scheduler@kubernetes \
40
  --cluster=kubernetes \
41
  --user=system:kube-scheduler \
42
  --kubeconfig=/opt/kubernetes/cfg/scheduler.kubeconfig
43

44
koevn@k8s-master-01:/opt/cert$ sudo kubectl config use-context system:kube-scheduler@kubernetes \
45
  --kubeconfig=/opt/kubernetes/cfg/scheduler.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo cat > admin-csr.json << EOF
2
{
3
  "CN": "admin",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "system:masters",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ]
17
}
18
EOF
19

20
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert \
21
  -ca=/opt/cert/kubernetes/pki/ca.pem \
22
  -ca-key=/opt/cert/kubernetes/pki/ca-key.pem \
23
  -config=ca-config.json \
24
  -profile=kubernetes \
25
  admin-csr.json | cfssljson -bare /opt/cert/kubernetes/pki/admin
26

27
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-cluster kubernetes \
28
  --certificate-authority=/opt/cert/kubernetes/pki/ca.pem \
29
  --embed-certs=true \
30
  --server=https://10.88.12.100:9443 \
31
  --kubeconfig=/opt/kubernetes/cfg/admin.kubeconfig
32

33
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-credentials kubernetes-admin \
34
  --client-certificate=/opt/cert/kubernetes/pki//admin.pem \
35
  --client-key=/opt/cert/kubernetes/pki//admin-key.pem \
36
  --embed-certs=true \
37
  --kubeconfig=/opt/kubernetes/cfg/admin.kubeconfig
38

39
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-context kubernetes-admin@kubernetes \
40
  --cluster=kubernetes \
41
  --user=kubernetes-admin \
42
  --kubeconfig=/opt/kubernetes/cfg/admin.kubeconfig
43

44
koevn@k8s-master-01:/opt/cert$ sudo kubectl config use-context kubernetes-admin@kubernetes \
45
--kubeconfig=/opt/kubernetes/cfg/admin.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo cat > kube-proxy-csr.json  << EOF
2
{
3
  "CN": "system:kube-proxy",
4
  "key": {
5
    "algo": "rsa",
6
    "size": 2048
7
  },
8
  "names": [
9
    {
10
      "C": "CN",
11
      "ST": "Beijing",
12
      "L": "Beijing",
13
      "O": "system:kube-proxy",
14
      "OU": "Kubernetes-manual"
15
    }
16
  ]
17
}
18
EOF
19

20
koevn@k8s-master-01:/opt/cert$ sudo cfssl gencert \
21
  -ca=/opt/cert/kubernetes/pki/ca.pem \
22
  -ca-key=/opt/cert/kubernetes/pki/ca-key.pem \
23
  -config=ca-config.json \
24
  -profile=kubernetes \
25
  kube-proxy-csr.json | cfssljson -bare /opt/kubernetes/cfg/kube-proxy
26

27
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-cluster kubernetes \
28
  --certificate-authority=/opt/cert/kubernetes/pki/ca.pem \
29
  --embed-certs=true \
30
  --server=https://10.88.12.100:9443 \
31
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig
32

33
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-credentials kube-proxy \
34
  --client-certificate=/opt/cert/kubernetes/pki/kube-proxy.pem \
35
  --client-key=/opt/cert/kubernetes/pki/kube-proxy-key.pem \
36
  --embed-certs=true \
37
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig
38

39
koevn@k8s-master-01:/opt/cert$ sudo kubectl config set-context kube-proxy@kubernetes \
40
  --cluster=kubernetes \
41
  --user=kube-proxy \
42
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig
43

44
koevn@k8s-master-01:/opt/cert$ sudo kubectl config use-context kube-proxy@kubernetes \
45
--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig

1
koevn@k8s-master-01:/opt/cert$ sudo openssl genrsa -out /opt/cert/kubernetes/pki/sa.key 2048
2
koevn@k8s-master-01:/opt/cert$ sudo openssl rsa -in /opt/cert/kubernetes/pki/sa.key \
3
-pubout -out /opt/cert/kubernetes/pki/sa.pub

1
koevn@k8s-master-01:~$ ls /opt/cert/kubernetes/pki
2
-rw-r--r-- 1 root root 1025 May 20 17:11 admin.csr
3
-rw------- 1 root root 1679 May 20 17:11 admin-key.pem
4
-rw-r--r-- 1 root root 1444 May 20 17:11 admin.pem
5
-rw-r--r-- 1 root root 1383 May 20 16:25 apiserver.csr
6
-rw------- 1 root root 1675 May 20 16:25 apiserver-key.pem
7
-rw-r--r-- 1 root root 1777 May 20 16:25 apiserver.pem
8
-rw-r--r-- 1 root root 1070 May 20 11:58 ca.csr
9
-rw------- 1 root root 1679 May 20 11:58 ca-key.pem
10
-rw-r--r-- 1 root root 1363 May 20 11:58 ca.pem
11
-rw-r--r-- 1 root root 1082 May 20 16:38 controller-manager.csr
12
-rw------- 1 root root 1679 May 20 16:38 controller-manager-key.pem
13
-rw-r--r-- 1 root root 1501 May 20 16:38 controller-manager.pem
14
-rw-r--r-- 1 root root  940 May 20 16:26 front-proxy-ca.csr
15
-rw------- 1 root root 1675 May 20 16:26 front-proxy-ca-key.pem
16
-rw-r--r-- 1 root root 1094 May 20 16:26 front-proxy-ca.pem
17
-rw-r--r-- 1 root root  903 May 20 16:30 front-proxy-client.csr
18
-rw------- 1 root root 1679 May 20 16:30 front-proxy-client-key.pem
19
-rw-r--r-- 1 root root 1188 May 20 16:30 front-proxy-client.pem
20
-rw-r--r-- 1 root root 1045 May 20 18:30 kube-proxy.csr
21
-rw------- 1 root root 1679 May 20 18:30 kube-proxy-key.pem
22
-rw-r--r-- 1 root root 1464 May 20 18:30 kube-proxy.pem
23
-rw------- 1 root root 1704 May 21 09:21 sa.key
24
-rw-r--r-- 1 root root  451 May 21 09:21 sa.pub
25
-rw-r--r-- 1 root root 1058 May 20 17:00 scheduler.csr
26
-rw------- 1 root root 1679 May 20 17:00 scheduler-key.pem
27
-rw-r--r-- 1 root root 1476 May 20 17:00 scheduler.pem
28

29
koevn@k8s-master-01:~$ ls /opt/cert/kubernetes/pki/ | wc -l  # 查看总数
30
26

1
koevn@k8s-master-01:~$ sudo mkdir -pv /opt/kubernetes/{bin,etc,cert,cfg}
2
koevn@k8s-master-01:~$ sudo tree /opt/kubernetes    # 查看文件层级
3
/opt/kubernetes
4
├── bin
5
│   ├── kube-apiserver
6
│   ├── kube-controller-manager
7
│   ├── kubectl
8
│   ├── kubelet
9
│   ├── kube-proxy
10
│   └── kube-scheduler
11
├── cert
12
│   ├── admin-key.pem
13
│   ├── admin.pem
14
│   ├── apiserver-key.pem
15
│   ├── apiserver.pem
16
│   ├── ca-key.pem
17
│   ├── ca.pem
18
│   ├── controller-manager-key.pem
19
│   ├── controller-manager.pem
20
│   ├── front-proxy-ca.pem
21
│   ├── front-proxy-client-key.pem
22
│   ├── front-proxy-client.pem
23
│   ├── kube-proxy-key.pem
24
│   ├── kube-proxy.pem
25
│   ├── sa.key
26
│   ├── sa.pub
27
│   ├── scheduler-key.pem
28
│   └── scheduler.pem
29
├── cfg
30
│   ├── admin.kubeconfig
31
│   ├── bootstrap-kubelet.kubeconfig
32
│   ├── bootstrap.secret.yaml
33
│   ├── controller-manager.kubeconfig
34
│   ├── kubelet.kubeconfig
35
│   ├── kube-proxy.kubeconfig
36
│   └── scheduler.kubeconfig
37
├── etc
38
│   ├── kubelet-conf.yml
39
│   └── kube-proxy.yaml
40
└── manifests
41

42
6 directories, 32 files

1
koevn@k8s-master-01:~$ sudo apt install haproxy

1
koevn@k8s-master-01:~$ sudo cat >/etc/haproxy/haproxy.cfg << EOF
2
global
3
 maxconn 2000
4
 ulimit-n 16384
5
 log 127.0.0.1 local0 err
6
 stats timeout 30s
7

8
defaults
9
 log global
10
 mode http
11
 option httplog
12
 timeout connect 5000
13
 timeout client 50000
14
 timeout server 50000
15
 timeout http-request 15s
16
 timeout http-keep-alive 15s
17

18
listen stats
19
 mode http
20
 bind 0.0.0.0:9999
21
 stats enable
22
 log global
23
 stats uri     /haproxy-status
24
 stats auth    admin:K123456
25

26
frontend monitor-in
27
 bind *:33305
28
 mode http
29
 option httplog
30
 monitor-uri /monitor
31

32
frontend k8s-master
33
 bind 0.0.0.0:9443
34
 bind 127.0.0.1:9443
35
 mode tcp
36
 option tcplog
37
 tcp-request inspect-delay 5s
38
 default_backend k8s-master
39

40

41
backend k8s-master
42
 mode tcp
43
 option tcplog
44
 option tcp-check
45
 balance roundrobin
46
 default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
47
 server k8s-master-01 10.88.12.60:6443 check
48
 server k8s-master-02 10.88.12.61:6443 check
49
 server k8s-master-03 10.88.12.62:6443 check
50
EOF

1
koevn@k8s-master-01:~$ sudo systemctl daemon-reload
2
koevn@k8s-master-01:~$ sudo systemctl enable --now haproxy.service

1
koevn@k8s-master-01:~$ sudo cat /etc/systemd/system/kube-apiserver.service << EOF
2
[Unit]
3
Description=Kubernetes API Server
4
Documentation=https://github.com/kubernetes/kubernetes
5
After=network.target
6

7
[Service]
8
ExecStart=/opt/kubernetes/bin/kube-apiserver \
9
      --v=2 \
10
      --allow-privileged=true \
11
      --bind-address=0.0.0.0 \
12
      --secure-port=6443 \
13
      --advertise-address=10.88.12.60 \    # 注意更改每个Master节点IP
14
      --service-cluster-ip-range=10.90.0.0/16 \  # 集群cluster网段根据使用配置
15
      --service-node-port-range=30000-32767 \
16
      --etcd-servers=https://10.88.12.60:2379,https://10.88.12.61:2379,https://10.88.12.62:2379 \
17
      --etcd-cafile=/opt/etcd/cert/etcd-ca.pem \
18
      --etcd-certfile=/opt/etcd/cert/etcd.pem \
19
      --etcd-keyfile=/opt/etcd/cert/etcd-key.pem \
20
      --client-ca-file=/opt/kubernetes/cert/ca.pem \
21
      --tls-cert-file=/opt/kubernetes/cert/apiserver.pem \
22
      --tls-private-key-file=/opt/kubernetes/cert/apiserver-key.pem \
23
      --kubelet-client-certificate=/opt/kubernetes/cert/apiserver.pem \
24
      --kubelet-client-key=/opt/kubernetes/cert/apiserver-key.pem \
25
      --service-account-key-file=/opt/kubernetes/cert/sa.pub \
26
      --service-account-signing-key-file=/opt/kubernetes/cert/sa.key \
27
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \
28
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
29
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
30
      --authorization-mode=Node,RBAC \
31
      --enable-bootstrap-token-auth=true \
32
      --requestheader-client-ca-file=/opt/kubernetes/cert/front-proxy-ca.pem \
33
      --proxy-client-cert-file=/opt/kubernetes/cert/front-proxy-client.pem \
34
      --proxy-client-key-file=/opt/kubernetes/cert/front-proxy-client-key.pem \
35
      --requestheader-allowed-names=aggregator \
36
      --requestheader-group-headers=X-Remote-Group \
37
      --requestheader-extra-headers-prefix=X-Remote-Extra- \
38
      --requestheader-username-headers=X-Remote-User \
39
      --enable-aggregator-routing=true
40
Restart=on-failure
41
RestartSec=10s
42
LimitNOFILE=65535
43

44
[Install]
45
WantedBy=multi-user.target
46
EOF

1
koevn@k8s-master-01:~$ sudo systemctl daemon-reload

1
koevn@k8s-master-01:~$ sudo systemctl enable --now kube-apiserver.service

1
koevn@k8s-master-01:~$ sudo systemctl status kube-apiserver.service

1
koevn@k8s-master-01:~$ sudo cat /etc/systemd/system/kube-controller-manager.service << EOF
2
[Unit]
3
Description=Kubernetes Controller Manager
4
Documentation=https://github.com/kubernetes/kubernetes
5
After=network.target
6

7
[Service]
8
ExecStart=/opt/kubernetes/bin/kube-controller-manager \
9
      --v=2 \
10
      --root-ca-file=/opt/kubernetes/cert/ca.pem \
11
      --cluster-signing-cert-file=/opt/kubernetes/cert/ca.pem \
12
      --cluster-signing-key-file=/opt/kubernetes/cert/ca-key.pem \
13
      --service-account-private-key-file=/opt/kubernetes/cert/sa.key \
14
      --kubeconfig=/opt/kubernetes/cfg/controller-manager.kubeconfig \
15
      --leader-elect=true \
16
      --use-service-account-credentials=true \
17
      --node-monitor-grace-period=40s \
18
      --node-monitor-period=5s \
19
      --controllers=*,bootstrapsigner,tokencleaner \
20
      --allocate-node-cidrs=true \
21
      --service-cluster-ip-range=10.90.0.0/16 \    # 集群cluster网段根据使用更改
22
      --cluster-cidr=10.100.0.0/16 \               # 集群pod网段根据使用更改
23
      --node-cidr-mask-size-ipv4=24 \              # 网段子网掩码设为24
24
      --requestheader-client-ca-file=/opt/kubernetes/cert/front-proxy-ca.pem
25

26
Restart=always
27
RestartSec=10s
28

29
[Install]
30
WantedBy=multi-user.target
31
EOF

1
koevn@k8s-master-01:~$ sudo systemctl daemon-reload

1
koevn@k8s-master-01:~$ sudo systemctl enable --now kube-controller-manager.service

1
koevn@k8s-master-01:~$ sudo systemctl status kube-controller-manager.service

1
koevn@k8s-master-01:~$ sudo cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
2
[Unit]
3
Description=Kubernetes Scheduler
4
Documentation=https://github.com/kubernetes/kubernetes
5
After=network.target
6

7
[Service]
8
ExecStart=/opt/kubernetes/bin/kube-scheduler \
9
      --v=2 \
10
      --bind-address=0.0.0.0 \
11
      --leader-elect=true \
12
      --kubeconfig=/opt/kubernetes/cfg/scheduler.kubeconfig
13

14
Restart=always
15
RestartSec=10s
16

17
[Install]
18
WantedBy=multi-user.target
19
EOF

1
koevn@k8s-master-01:~$ sudo systemctl daemon-reload

1
koevn@k8s-master-01:~$ sudo systemctl enable --now kube-scheduler.service

1
koevn@k8s-master-01:~$ sudo systemctl status kube-scheduler.service

1
koevn@k8s-master-01:~$ sudo kubectl config set-cluster kubernetes \
2
--certificate-authority=/opt/cert/kubernetes/pki/ca.pem \
3
--embed-certs=true \
4
--server=https://10.88.12.100:9443 \
5
--kubeconfig=/opt/kubernetes/cfg/bootstrap-kubelet.kubeconfig

1
koevn@k8s-master-01:~$ sudo echo "$(head -c 6 /dev/urandom | md5sum | head -c 6)"."$(head -c 16 /dev/urandom | md5sum | head -c 16)"
2
79b841.0677456fb3b47289

1
koevn@k8s-master-01:~$ sudo kubectl config set-credentials tls-bootstrap-token-user \
2
--token=79b841.0677456fb3b47289 \
3
--kubeconfig=/opt/kubernetes/cfg/bootstrap-kubelet.kubeconfig

1
koevn@k8s-master-01:~$ sudo kubectl config set-context tls-bootstrap-token-user@kubernetes \
2
--cluster=kubernetes \
3
--user=tls-bootstrap-token-user \
4
--kubeconfig=/opt/kubernetes/cfg/bootstrap-kubelet.kubeconfig
5

6
koevn@k8s-master-01:~$ sudo kubectl config use-context tls-bootstrap-token-user@kubernetes \
7
--kubeconfig=/opt/kubernetes/cfg/bootstrap-kubelet.kubeconfig

1
mkdir -pv /home/koevn/.kube && cp /opt/kubernetes/cfg/admin.kubeconfig /home/koevn/.kube/config

1
koevn@k8s-master-01:~$ sudo kubectl get cs   # 查看 Kubernetes 控制平面核心组件的健康状态
2
Warning: v1 ComponentStatus is deprecated in v1.19+
3
NAME                 STATUS    MESSAGE   ERROR
4
etcd-0               Healthy   ok
5
controller-manager   Healthy   ok
6
scheduler            Healthy   ok

1
koevn@k8s-master-01:~$ sudo cat > bootstrap.secret.yaml << EOF
2
apiVersion: v1
3
kind: Secret
4
metadata:
5
  name: bootstrap-token-79b841
6
  namespace: kube-system
7
type: bootstrap.kubernetes.io/token
8
stringData:
9
  description: "The default bootstrap token generated by 'kubelet '."
10
  token-id: 79b841
11
  token-secret: 0677456fb3b47289
12
  usage-bootstrap-authentication: "true"
13
  usage-bootstrap-signing: "true"
14
  auth-extra-groups:  system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress
15

16
---
17
apiVersion: rbac.authorization.k8s.io/v1
18
kind: ClusterRoleBinding
19
metadata:
20
  name: kubelet-bootstrap
21
roleRef:
22
  apiGroup: rbac.authorization.k8s.io
23
  kind: ClusterRole
24
  name: system:node-bootstrapper
25
subjects:
26
- apiGroup: rbac.authorization.k8s.io
27
  kind: Group
28
  name: system:bootstrappers:default-node-token
29
---
30
apiVersion: rbac.authorization.k8s.io/v1
31
kind: ClusterRoleBinding
32
metadata:
33
  name: node-autoapprove-bootstrap
34
roleRef:
35
  apiGroup: rbac.authorization.k8s.io
36
  kind: ClusterRole
37
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
38
subjects:
39
- apiGroup: rbac.authorization.k8s.io
40
  kind: Group
41
  name: system:bootstrappers:default-node-token
42
---
43
apiVersion: rbac.authorization.k8s.io/v1
44
kind: ClusterRoleBinding
45
metadata:
46
  name: node-autoapprove-certificate-rotation
47
roleRef:
48
  apiGroup: rbac.authorization.k8s.io
49
  kind: ClusterRole
50
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
51
subjects:
52
- apiGroup: rbac.authorization.k8s.io
53
  kind: Group
54
  name: system:nodes
55
---
56
apiVersion: rbac.authorization.k8s.io/v1
57
kind: ClusterRole
58
metadata:
59
  annotations:
60
    rbac.authorization.kubernetes.io/autoupdate: "true"
61
  labels:
62
    kubernetes.io/bootstrapping: rbac-defaults
63
  name: system:kube-apiserver-to-kubelet
64
rules:
65
  - apiGroups:
66
      - ""
67
    resources:
68
      - nodes/proxy
69
      - nodes/stats
70
      - nodes/log
71
      - nodes/spec
72
      - nodes/metrics
73
    verbs:
74
      - "*"
75
---
76
apiVersion: rbac.authorization.k8s.io/v1
77
kind: ClusterRoleBinding
78
metadata:
79
  name: system:kube-apiserver
80
  namespace: ""
81
roleRef:
82
  apiGroup: rbac.authorization.k8s.io
83
  kind: ClusterRole
84
  name: system:kube-apiserver-to-kubelet
85
subjects:
86
  - apiGroup: rbac.authorization.k8s.io
87
    kind: User
88
    name: kube-apiserver

1
kubectl create -f bootstrap.secret.yaml

1
koevn@k8s-worker-01:~$ sudo mkdir -pv /opt/kubernetes/{bin,cert,cfg,etc}
2
koevn@k8s-worker-01:~$ sudo tree /opt/kubernetes/   # 查看目录结构
3
/opt/kubernetes/
4
├── bin
5
│   ├── kubectl
6
│   ├── kubelet
7
│   └── kube-proxy
8
├── cert             # 证书文件从master节点copy过来
9
│   ├── ca.pem
10
│   ├── front-proxy-ca.pem
11
│   ├── kube-proxy-key.pem
12
│   └── kube-proxy.pem
13
├── cfg              # kubeconfig文件从master节点copy过来
14
│   ├── bootstrap-kubelet.kubeconfig
15
│   ├── kubelet.kubeconfig
16
│   └── kube-proxy.kubeconfig
17
├── etc
18
│   ├── kubelet-conf.yml
19
│   └── kube-proxy.yaml
20
└── manifests
21

22
6 directories, 12 files

1
koevn@k8s-worker-01:~$ sudo cat > /usr/lib/systemd/system/kubelet.service << EOF
2
[Unit]
3
Description=Kubernetes Kubelet
4
Documentation=https://github.com/kubernetes/kubernetes
5
After=network-online.target firewalld.service containerd.service
6
Wants=network-online.target
7
Requires=containerd.service
8

9
[Service]
10
ExecStart=/opt/kubernetes/bin/kubelet \
11
    --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap-kubelet.kubeconfig \
12
    --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
13
    --config=/opt/kubernetes/etc/kubelet-conf.yml \
14
    --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
15
    --node-labels=node.kubernetes.io/node=
16

17
[Install]
18
WantedBy=multi-user.target
19
EOF