GCDW是GBase 的云数仓版本(GBase Cloud Database Warehouse)， 其运行在k8s上，镜像要先上传到镜像仓库，在安装部署时各个节点再从镜像仓库同时拉取。本文介绍harbor安装配置方法，特别是启用https服务的方法。
harbor需要容器环境
yum install docker-ce docker-ce-cli containerd.io docker-compose-plugin还需要docker-compose,这个直接下载，就是个可执行文件，改名到/usr/local/bin下就可以了
[root@vm249 harbor]# ll /usr/local/bin/docker-compose
-rwxr-xr-x. 1 root root 44953600 Feb 21 00:58 /usr/local/bin/docker-compose
[root@vm249 harbor]#生成CA证书
如果不开通https服务，可以直接跳到后面部分
生成CA私钥
[root@vm249 ssl]# openssl genrsa -out ca.key 4096
Generating RSA private key, 4096 bit long modulus
.........................................++
........................................................................................................................................++
e is 65537 (0x10001)
[root@vm249 ssl]# ll
total 4
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
[root@vm249 ssl]#
生成CA证书 Generate the CA certificate.
openssl req -x509 -new -nodes -sha512 -days 3650 \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \
-key ca.key \
-out ca.crt其中
- -subj 指定组织名称等，如果使用FQDN连接Harbor主机，则必须将其指定为通用名称（
CN）属性，比如你的主机名或者域名等。 - -key 是前面生成的CA私钥
- -out 是生成的CA证书名字
openssl req -x509 -new -nodes -sha512 -days 3650 \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=172.16.3.249" \
-key ca.key \
-out ca.crt生成服务器端证书Generate a Server Certificate
生成私钥
openssl genrsa -out yourdomain.com.key 4096其中域名是yourdomain.com， 下面你是用IP的例子
[root@vm249 ssl]# openssl genrsa -out 172.16.3.249.key 4096
Generating RSA private key, 4096 bit long modulus
................................................++
.++
e is 65537 (0x10001)
[root@vm249 ssl]# ll
total 12
-rw-r--r--. 1 root root 3247 Feb 21 17:49 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 17:33 ca.crt
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
[root@vm249 ssl]#
生成CSR Generate a certificate signing request (CSR).
openssl req -sha512 -new \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \
-key yourdomain.com.key \
-out yourdomain.com.csr如下是用IP作为主机名
[root@vm249 ssl]# openssl req -sha512 -new \
> -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=172.16.3.249" \
> -key 172.16.3.249.key \
> -out 172.16.3.249.csr
[root@vm249 ssl]# ll
total 16
-rw-r--r--. 1 root root 1704 Feb 21 18:10 172.16.3.249.csr
-rw-r--r--. 1 root root 3247 Feb 21 17:49 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 17:33 ca.crt
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
[root@vm249 ssl]#生成X509 v3的扩展文件 Generate an x509 v3 extension file.
包括域名和IP方式的主机名，用于harbor编译时指定了SAN和X509 v3（Harbor host that complies with the Subject Alternative Name (SAN) and x509 v3 extension requirements.）
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1=yourdomain.com
DNS.2=yourdomain
DNS.3=hostname
EOF如果用IP,则需要在subjectAltName部分直接写IP
[root@vm249 ssl]# cat > v3.ext <<-EOF
> authorityKeyIdentifier=keyid,issuer
> basicConstraints=CA:FALSE
> keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
> extendedKeyUsage = serverAuth
> subjectAltName = IP:172.16.3.249
> EOF
[root@vm249 ssl]# ll
total 20
-rw-r--r--. 1 root root 1704 Feb 21 18:10 172.16.3.249.csr
-rw-r--r--. 1 root root 3247 Feb 21 17:49 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 17:33 ca.crt
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
-rw-r--r--. 1 root root 231 Feb 21 18:32 v3.ext
[root@vm249 ssl]# cat v3.ext
authorityKeyIdentifier=keyid,issuer
[root@vm249 ssl]# cat v3.ext
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = IP:172.16.3.249
[root@vm249 ssl]#
用v3.ext生成harbor主机的证书 Use the v3.ext file to generate a certificate for your Harbor host.
openssl x509 -req -sha512 -days 3650 \
-extfile v3.ext \
-CA ca.crt -CAkey ca.key -CAcreateserial \
-in yourdomain.com.csr \
-out yourdomain.com.crt如下是使用IP的例子
openssl x509 -req -sha512 -days 3650 \
-extfile v3.ext \
-CA ca.crt -CAkey ca.key -CAcreateserial \
-in 172.16.3.249.csr \
-out 172.16.3.249.crt[root@vm249 ssl]# openssl x509 -req -sha512 -days 3650 \
> -extfile v3.ext \
> -CA ca.crt -CAkey ca.key -CAcreateserial \
> -in 172.16.3.249.csr \
> -out 172.16.3.249.crt
Signature ok
subject=/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=172.16.3.249
Getting CA Private Key
[root@vm249 ssl]#
[root@vm249 ssl]# ll
total 28
-rw-r--r--. 1 root root 2061 Feb 21 18:34 172.16.3.249.crt
-rw-r--r--. 1 root root 1704 Feb 21 18:10 172.16.3.249.csr
-rw-r--r--. 1 root root 3247 Feb 21 17:49 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 17:33 ca.crt
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
-rw-r--r--. 1 root root 17 Feb 21 18:34 ca.srl
-rw-r--r--. 1 root root 231 Feb 21 18:32 v3.ext
[root@vm249 ssl]#
将证书发给容器和harbor
将服务器端证书和私钥复制到harbor主机
Copy the server certificate and key into the certficates folder on your Harbor host.
cp yourdomain.com.crt /data/cert/
cp yourdomain.com.key /data/cert/如下是IP的操作记录。 目录如果不存在，可以创建上。
[root@vm249 ssl]# mkdir -p /data/cert
[root@vm249 ssl]# cp 172.16.3.249.crt /data/cert/
[root@vm249 ssl]# cp 172.16.3.249.key /data/cert/
[root@vm249 ssl]# ll /data/cert/
total 8
-rw-r--r--. 1 root root 2061 Feb 21 19:13 172.16.3.249.crt
-rw-r--r--. 1 root root 3247 Feb 21 19:13 172.16.3.249.key
[root@vm249 ssl]#
将crt转化为cert,提供给docker使用
Convert yourdomain.com.crt to yourdomain.com.cert, for use by Docker.
The Docker daemon interprets .crt files as CA certificates and .cert files as client certificates.
docker进程将crt作为CA证书， cert作为客户端证书。
openssl x509 -inform PEM -in yourdomain.com.crt -out yourdomain.com.cert运行记录
[root@vm249 ssl]# openssl x509 -inform PEM -in 172.16.3.249.crt -out 172.16.3.249.cert
[root@vm249 ssl]# ll
total 32
-rw-r--r--. 1 root root 2061 Feb 21 19:17 172.16.3.249.cert
-rw-r--r--. 1 root root 2061 Feb 21 18:34 172.16.3.249.crt
-rw-r--r--. 1 root root 1704 Feb 21 18:10 172.16.3.249.csr
-rw-r--r--. 1 root root 3247 Feb 21 17:49 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 17:33 ca.crt
-rw-r--r--. 1 root root 3247 Feb 21 17:23 ca.key
-rw-r--r--. 1 root root 17 Feb 21 18:34 ca.srl
-rw-r--r--. 1 root root 231 Feb 21 18:32 v3.ext
[root@vm249 ssl]#
将服务器端证书，私钥和CA文件，复制到docker的harbor主机目录下。
Copy the server certificate, key and CA files into the Docker certificates folder on the Harbor host. You must create the appropriate folders first.
注意，默认是443端口，如果不是默认值，则后面创建的目录，要带上端口号。 比如有从172.16.3.249，变成172.16.3.249:8443
cp yourdomain.com.cert /etc/docker/certs.d/yourdomain.com/
cp yourdomain.com.key /etc/docker/certs.d/yourdomain.com/
cp ca.crt /etc/docker/certs.d/yourdomain.com/运行记录
[root@vm249 ssl]# mkdir -p /etc/docker/certs.d/172.16.3.249/
[root@vm249 ssl]# cp 172.16.3.249.cert /etc/docker/certs.d/172.16.3.249/
[root@vm249 ssl]# cp 172.16.3.249.key /etc/docker/certs.d/172.16.3.249/
[root@vm249 ssl]# cp ca.crt /etc/docker/certs.d/172.16.3.249/
[root@vm249 ssl]# ll /etc/docker/certs.d/172.16.3.249/
total 12
-rw-r--r--. 1 root root 2061 Feb 21 19:20 172.16.3.249.cert
-rw-r--r--. 1 root root 3247 Feb 21 19:20 172.16.3.249.key
-rw-r--r--. 1 root root 2029 Feb 21 19:20 ca.crt
[root@vm249 ssl]#
如果修改了SSL端口
如果不是默认的443,则前面创建的目录，要加上端口,比如。 并且所有的CA文件也是要放到这个目录下。
[root@vm249 ssl]# mkdir /etc/docker/certs.d/172.16.3.249:8443
重启容器docker
systemctl restart docker修改harbor支持https
修改配置文件
包括 hostname 和 https 部分，其中证书和私钥正确配置。 如果不开通https服务，可以注释掉相关部分。 注意配置文件缩进格式，不要出现无效的额外空格等，否则会造成解析错误。
[root@vm249 ssl]# cat ../harbor.yml
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 172.16.3.249
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 8088
# https related config
https:
# https port for harbor, default is 443
port: 8443
# The path of cert and key files for nginx
certificate: /data/cert/172.16.3.249.crt
private_key: /data/cert/172.16.3.249.key
。。。。。。启动服务
如果没有运行在http模式下，可以直接install.sh安装
./install.sh如果已经部署了http模式，可以重新部署
./prepare
docker-compose down -v
docker-compose up -d执行记录
[root@vm249 harbor]# ./prepare
prepare base dir is set to /root/harbor
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[root@vm249 harbor]# ll -rt
total 56
-rwxr-xr-x. 1 root root 1881 Dec 14 20:29 prepare
-rwxr-xr-x. 1 root root 3171 Dec 14 20:29 install.sh
-rw-r--r--. 1 root root 11567 Dec 14 20:29 harbor.yml.tmpl
-rw-r--r--. 1 root root 3639 Dec 14 20:29 common.sh
-rw-r--r--. 1 root root 11347 Dec 14 20:29 LICENSE
drwxr-xr-x. 3 root root 20 Feb 21 00:33 common
drwxr-xr-x. 2 root root 159 Feb 21 19:17 ssl
-rw-r--r--. 1 root root 11576 Feb 21 21:42 harbor.yml
-rw-r--r--. 1 root root 5947 Feb 21 21:47 docker-compose.yml
[root@vm249 harbor]# systemctl daemon-reload
[root@vm249 harbor]# docker-compose down -v^C
[root@vm249 harbor]# systemctl restart docker
[root@vm249 harbor]# docker-compose down -v
[+] Running 10/10
⠿ Container harbor-jobservice Removed 11.3s
⠿ Container registryctl Removed 10.4s
⠿ Container nginx Removed 1.3s
⠿ Container harbor-portal Removed 0.3s
⠿ Container harbor-core Removed 3.4s
⠿ Container harbor-db Removed 1.0s
⠿ Container registry Removed 0.9s
⠿ Container redis Removed 1.0s
⠿ Container harbor-log Removed 10.3s
⠿ Network harbor_harbor Removed 0.2s
[root@vm249 harbor]# docker-compose down -v^C
[root@vm249 harbor]# docker-compose up -d
[+] Running 10/10
⠿ Network harbor_harbor Created 0.3s
⠿ Container harbor-log Started 2.2s
⠿ Container registry Started 5.4s
⠿ Container harbor-db Started 6.2s
⠿ Container registryctl Started 5.9s
⠿ Container harbor-portal Started 6.1s
⠿ Container redis Started 5.8s
⠿ Container harbor-core Started 6.8s
⠿ Container nginx Started 9.2s
⠿ Container harbor-jobservice Started 9.1s
[root@vm249 harbor]#
测试效果
如果访问的是8088端口，也会自动转到8443的https。
增加https的docker daemon.json配置
[root@vm249 harbor]# vi /etc/docker/daemon.json
[root@vm249 harbor]# cat /etc/docker/daemon.json
{
"registry-mirrors": [
"https://registry.docker-cn.com",
"http://hub-mirror.c.163.com",
"https://docker.mirrors.ustc.edu.cn",
"https://172.16.3.249:8443"
],
"insecure-registries": [
],
"log-opts": {
"max-size": "10m"
}
}
[root@vm249 harbor]#
[root@vm249 harbor]# systemctl daemon-reload
[root@vm249 harbor]# systemctl restart docker
[root@vm249 harbor]# systemctl restart harbor
[root@vm249 harbor]# docker login 172.16.3.249:8443
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@vm249 harbor]#
其它节点远程访问harbor的方法
需要将所需的/etc/docker/cert.d下对应的地址的证书复制过去。 其中的daemon.json，根据需要进行修改。 如果目录不存在，可以提前创建。
[root@k8s-81 ~]# docker login https://172.16.3.249:8443
Username: admin
Password:
Error response from daemon: Get "https://172.16.3.249:8443/v2/": x509: certificate signed by unknown authority
[root@k8s-81 ~]# cd /etc/docker/
[root@k8s-81 docker]# ll
total 0
[root@k8s-81 docker]# scp -r 172.16.3.249:/etc/docker/cert.d/172.16.3.249:8443 ./certd.d/
The authenticity of host '172.16.3.249 (172.16.3.249)' can't be established.
ECDSA key fingerprint is SHA256:Xs1gi6NKPEsAxLRIL2NHIv7jG1vt68oBlWZ0YUe/Swk.
ECDSA key fingerprint is MD5:b4:9c:dd:e1:3c:42:28:8d:db:c5:a0:73:30:2f:60:78.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '172.16.3.249' (ECDSA) to the list of known hosts.
root@172.16.3.249's password:
Permission denied, please try again.
root@172.16.3.249's password:
172.16.3.249.cert 100% 2053 36.5KB/s 00:00
172.16.3.249.crt 100% 2053 44.7KB/s 00:00
172.16.3.249.key 100% 3247 597.2KB/s 00:00
ca.crt 100% 2029 29.0KB/s 00:00
daemon.json 100% 276 42.9KB/s 00:00
[root@k8s-81 docker]# ll
total 4
drwxr-xr-x. 3 root root 31 Feb 23 08:59 certs.d
-rw-r--r--. 1 root root 276 Feb 23 08:59 daemon.json
[root@k8s-81 docker]# cat daemon.json
{
"registry-mirrors": [
"https://registry.docker-cn.com",
"http://hub-mirror.c.163.com",
"https://docker.mirrors.ustc.edu.cn",
"https://172.16.3.249:8443"
],
"insecure-registries": [
],
"log-opts": {
"max-size": "10m"
}
}
[root@k8s-81 docker]# ll
total 4
drwxr-xr-x. 3 root root 31 Feb 23 08:59 certs.d
-rw-r--r--. 1 root root 276 Feb 23 08:59 daemon.json
[root@k8s-81 docker]# cd certs.d/
[root@k8s-81 certs.d]# ll
total 0
drwxr-xr-x. 2 root root 93 Feb 23 08:59 172.16.3.249:8443
[root@k8s-81 certs.d]# systemctl daemon-reload
[root@k8s-81 certs.d]# docker login https://172.16.3.249:8443
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@k8s-81 certs.d]# ll
total 0
drwxr-xr-x. 2 root root 93 Feb 23 08:59 172.16.3.249:8443
[root@k8s-81 certs.d]# cd ..
[root@k8s-81 docker]# ll
total 4
drwxr-xr-x. 3 root root 31 Feb 23 08:59 certs.d
-rw-r--r--. 1 root root 276 Feb 23 08:59 daemon.json
[root@k8s-81 docker]#
参考
https://goharbor.io/docs/2.0.0/install-config/configure-https/
