OS
PaaS私有化产品部署离线安装源适配问题解决指南
前言 最新在适配我司产品在信创操作系统各个版本上的安装，在踩了很多坑后， 总结出了一套有效的适配流程，在此记录一下。 由于信创国产化操作系统都是基于openEuler系统或Centos系统来的，包括系统软件包名字，release号等都修改了，在信创服务器上安装产品其实最大的工作量是需要适配跟系统相关的软件，和编译arm64版的二进制文件，尤其前一个比较难弄，因为有很多信创系统和发行版本，我们的目标是， 利用一台x86_64或者arm64架构的机器来构建所有的软件源，基本思路就是： 由于官方没有信创docker镜像，需要自己做，首先需要制作信创os docker镜像的流程是： 启动一个容器做构建环境(如:centos) –> 配置官方源 –> 通过yum –installroot=… 来创建基本的系统目录 –> 通过tar打包系统 –> docker import做成镜像 docker run起来信创系统docker镜像用作构建环境—> 配置官方源 —通过repotrack下载–> 打包成nexus 适配细节 首先是制作os docker镜像，下面是制作脚本：
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 |
#!/usr/bin/env bash # 用法： build.sh iso文件路径 # 例如：build.sh /data/build/os_images/Kylin-Server-V10-SP3-General-Release-2303-ARM64.iso set -e set -o pipefail current_dir=$( cd "$(dirname "$0")" pwd ) source $current_dir/tools/color.sh BUILD_TOOLS="vim vi yum net-tools curl dnf-utils createrepo iproute" function check_command(){ local r=0 `command -v $1 >/dev/null 2>&1` || r=1 return ${r} } install_docker(){ rpm -qa|grep podman > /dev/null 2>&1 && yum remove -y podman if [ "$ARCH" == 'amd64' ]; then centos_repo='Centos-7.repo'; else centos_repo='Centos-altarch-7.repo'; fi curl -o /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo sed -i -e 's/$releasever/7/g' -e 's/https/http/g' /etc/yum.repos.d/docker-ce.repo curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/${centos_repo} sed -i -e 's/$releasever/7/g' -e 's/https/http/g' /etc/yum.repos.d/CentOS-Base.repo yum makecache yum install docker-ce docker-ce-cli containerd.io -y if [ $? -eq 0 ]; then mkdir -p /etc/systemd/system/docker.service.d cat > /etc/systemd/system/docker.service.d/docker-options.conf << 'EOF' [Service] Environment="DOCKER_OPTS= --iptables=true \ --insecure-registry=0.0.0.0/0 \ --registry-mirror=https://dockerhub.xxx.com \ --registry-mirror=https://registry.docker-cn.com \ --registry-mirror=https://mirror.aliyuncs.com \ --registry-mirror=https://docker.mirrors.ustc.edu.cn \ --data-root=/data/docker \ --log-opt max-size=50m --log-opt max-file=5" EOF cat > /etc/systemd/system/docker.service << 'EOF' [Unit] Description=Docker Application Container Engine Documentation=http://docs.docker.com [Service] Type=notify Environment="DOCKER_STORAGE_OPTIONS=-s overlay2" Environment=GOTRACEBACK=crash ExecReload=/bin/kill -s HUP $MAINPID ExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT Delegate=yes KillMode=process ExecStart=/usr/bin/dockerd \ $DOCKER_OPTS \ $DOCKER_STORAGE_OPTIONS \ $DOCKER_DNS_OPTIONS TasksMax=infinity LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity TimeoutStartSec=1min # restart the docker process if it exits prematurely Restart=on-failure StartLimitBurst=3 StartLimitInterval=60s [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable docker systemctl start docker fi } create_multi_platform(){ if ! sudo docker ps | grep buildkitd >/dev/null 2>&1 || ! sudo docker buildx ls|grep 'multi-platform'>/dev/null 2>&1; then sudo docker run -it --rm --privileged tonistiigi/binfmt --install all if [ ! -f ~/.config/buildkit/buildkitd.toml ]; then sudo mkdir -p ~/.config/buildkit/ echo 'debug = true' > ~/.config/buildkit/buildkitd.toml echo 'insecure-entitlements = [ "network.host", "security.insecure" ]' >> ~/.config/buildkit/buildkitd.toml echo '[registry."dockerhub.xxx.com"]' >> ~/.config/buildkit/buildkitd.toml echo ' http = false' >> ~/.config/buildkit/buildkitd.toml echo ' insecure = true' >> ~/.config/buildkit/buildkitd.toml fi sudo docker buildx create \ --name multi-platform \ --use \ --platform linux/amd64,linux/arm64 \ --driver docker-container \ --driver-opt network=host \ --buildkitd-flags '--allow-insecure-entitlement network.host' \ --config ~/.config/buildkit/buildkitd.toml fi } main(){ cd ${current_dir} image_repo=$(find repos/online -name "*$image_name*$image_version*.repo") for arch in $image_arch; do if [ ! -z "$iso_path" ]; then iso_file=$(basename $iso_path) iso_dir=$(echo $iso_file|sed 's/.iso//g') iso_mount_dir=/mnt/$iso_dir [ ! -d "$iso_mount_dir" ] && mkdir -p $iso_mount_dir || (umount -f $iso_mount_dir || true) mount "$iso_path" $iso_mount_dir cat > repos/local.repo << EOF [local] name=local repo baseurl= file://$iso_mount_dir enable=1 gpgcheck=0 priority=1 EOF fi target=${image_name}_${image_version}_${arch} target_image=${image_name}:${image_version}-${arch} # docker pull --platform linux/$arch centos # docker pull --platform linux/$arch dockerhub.xxx.com/os/centos:8 docker pull dockerhub.xxx.com/os/centos:8-${arch} docker rm -f centos_${target} > /dev/null 2>&1 || true if [ ! -z "$iso_path" ]; then extra_arg="-v $iso_mount_dir:$iso_mount_dir" fi docker run -d --name centos_${target} --network host \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ${current_dir}:/opt/os \ ${extra_arg} \ dockerhub.xxx.com/os/centos:8-${arch} \ sleep 100000 docker exec centos_${target} bash -c "\ rm -rf /etc/yum.repos.d/*; \ if [ ! -z "$iso_path" ]; then \ cp /opt/os/repos/local.repo /etc/yum.repos.d/local.repo; \ fi; \ cp /opt/os/$image_repo /etc/yum.repos.d/$(basename $image_repo); \ if [ "$image_name" == "kylin" ]; then \ cp /opt/os/repos/online/kylin/RPM-GPG-KEY-kylin /etc/pki/rpm-gpg/RPM-GPG-KEY-kylin; \ elif [ "$image_name" == "uos" ]; then \ mkdir -p /opt/os/${target}/etc/dnf/vars;\ cp /opt/os/repos/online/uos/uos-auth-u /etc/dnf/vars/uos-auth-u; \ cp /opt/os/repos/online/uos/uos-auth-p /etc/dnf/vars/uos-auth-p; \ cp /opt/os/repos/online/uos/uos-auth-u /opt/os/${target}/etc/dnf/vars/uos-auth-u; \ cp /opt/os/repos/online/uos/uos-auth-p /opt/os/${target}/etc/dnf/vars/uos-auth-p; \ if [ "$image_version" == "20-1060e" ]; then \ echo 'ufu' > /etc/dnf/vars/StateMode; \ echo 'ufu' > /opt/os/${target}/etc/dnf/vars/StateMode; \ fi; \ fi; \ yum clean all; yum makecache; \ yum -y --installroot=/opt/os/${target} install $BUILD_TOOLS \ " cp /etc/skel/.bash* $current_dir/${target}/root/ echo "" > $current_dir/${target}/root/.bash_history rm -rf $current_dir/${target}/core.* tar -cvpf ${target}.tar --directory=$current_dir/${target} --exclude=proc --exclude=sys --exclude=dev --exclude=run --exclude=boot . # check_command docker || install_docker # create_multi_platform cat ${target}.tar | docker import - $target_image #5. 运行镜像，测试镜像是否能正常使用 docker run --rm --tty $target_image /bin/bash -c 'echo $(grep 'PRETTY_NAME' /etc/os-release) $(uname -m) builded success!!!' # clean docker rm -f centos_${target} umount -f $iso_mount_dir rm -rf ${target} ${target}.tar if [ $? -ne 0 ]; then errorlog "$target_image build Failed!!!!" exit else successlog "$target_image build successfull !" fi done if [ "$push" == "true" ]; then for arch in $image_arch; do registry_image="$registry/${image_name}:${image_version}" docker tag ${image_name}:${image_version}-${arch} ${registry_image}-${arch} docker push ${registry_image}-${arch} done if [[ "$image_arch" == "amd64 arm64" ]] || \ (docker images|awk '{print $1":"$2}'|grep ${registry_image}-amd64 >/dev/null 2>&1 && \ docker images|awk '{print $1":"$2}'|grep ${registry_image}-arm64 >/dev/null 2>&1); then docker manifest rm ${registry_image} || true docker manifest create --insecure ${registry_image} ${registry_image}-amd64 ${registry_image}-arm64 docker manifest annotate ${registry_image} ${registry_image}-amd64 --os linux --arch amd64 docker manifest annotate ${registry_image} ${registry_image}-arm64 --os linux --arch arm64 --variant v8 docker manifest push --insecure ${registry_image} if [ $? -ne 0 ]; then errorlog "${registry_image} build Failed!!!!" exit else successlog "${registry_image} build successfull !" fi fi fi } # 报错：x509: certificate has expired or is not yet valid，解决：docker.service.d/docker-options.conf里添加：--registry-mirror=https://dockerhub.xxx.com function Usage() { INFO="\ 功能:构建操作系统docker镜像 用法:$0 [选项] ... [参数] ... 支持参数: --image string [必选参数]镜像名称,如kylin:v10-sp3-arm64,openeuler:20.03-sp3-amd64,如果后面不带架构,如kylin:v10-sp3,则默认同时构建双架构 --image_arch string [可选参数]要构建的镜像的架构,默认是'amd64 arm64' --iso_path string [可选参数]不提供此参数,默认从在线官方软件源制作;如提供此参数,则默认以挂载iso文件作为软件源安装软件,如:/data/Kylin-Server-V10-SP3-General-Release-2303-ARM64.iso --registry string [可选参数]镜像仓库地址,比如要推送到dockerhub.xxx.com/os/kylin:v10-sp3,则值为dockerhub.xxx.com/os --push [可选参数]是否推送至镜像仓库,默认false --help 帮助文档 " infolog "$INFO" } ARGS=`getopt -o vh --long image:,image_arch:,iso_path:,push,help -- "$@"` ([[ $? -ne 0 ]] || [[ -z "$@" ]]) && Usage && exit eval set -- "$ARGS" while true; do case "$1" in --image) case "$2" in "") exit 1 ;; *) image=$2; shift 2 ;; esac ;; --image_arch) case "$2" in "") exit 1 ;; *) image_arch=$2; shift 2 ;; esac ;; --push) push='true';shift ;; --iso_path) case "$2" in "") exit 1 ;; *) iso_path=$2; shift 2 ;; esac ;; --registry) case "$2" in "") exit 1 ;; *) registry=$2; shift 2 ;; esac ;; -h|--help|-help) Usage; exit 0 ;; --) shift; break ;; *) warnlog "Invalid option, use -h or --help to get help message."; exit 1 ;; esac done : ${image_arch:="amd64 arm64"} : ${push:=false} : ${registry:="dockerhub.xxx.com/os"} image_name=$(echo $image|awk -F':' '{print $1}') image_tag=$(echo $image|awk -F':' '{print $2}') image_version=$(echo $image_tag|awk '{gsub(/-x86_64|amd64|-arm64|-aarch64/, "");print}') if [[ $? == 0 ]] ; then main else Usage exit 1 fi |
脚本逻辑解析： 由于一些信创操作系统中安装软件时需要认证机制，所以需要提前将认证放入对应目录(可以从安装好的os虚机中获取),此认证文件经过测试可以用到其它发行版本上，所以以后版本更新不用再更新此文件 脚本参数解析：
|
1 2 3 4 5 6 7 8 9 10 11 12 |
功能:构建操作系统docker镜像 ./mkimage.sh --help 用法:mkimage.sh [选项] ... [参数] ... 支持参数: --image string [必选参数]镜像名称， 如kylin:v10-sp3-arm64， openeuler:20.03-sp3-amd64, 如果后面不带架构，如kylin:v10-sp3，则默认同时构建双架构 --image_arch string [可选参数]要构建的镜像的架构,默认是'amd64 arm64' --iso_path string [可选参数]不提供此参数，默认从在线官方软件源制作；如提供此参数，则默认以挂载iso文件作为软件源安装软件,如：/data/Kylin-Server-V10-SP3-General-Release-2303-ARM64.iso --registry string [可选参数]镜像仓库地址，比如要推送到dockerhub.yonyoucloud.com/os/kylin:v10-sp3, 则值为dockerhub.yonyoucloud.com/os --push [可选参数]是否推送至镜像仓库,默认false --help 帮助文档 |
–image_arch: 构建的架构，默认两种架构同时构建 –iso_path: 官方发行版.iso后缀的镜像文件地址，如果提供此参数，则os镜像从.iso文件中的repo源进行制作，构建出来的镜像软件版本与.iso文件中的版本保持一致，如果不提供此参数， 则脚本默认会从官方在线的repo软件源中去下载发行版对应的软件版本，可能会存在某些软件跟.iso文件中不一致(经过测试，仅中科方德安装openssl时出现了依赖不一致) 定义软件清单 下载离线源时，通过定义一下packages.yaml文件，来定义全部所需的软件包： packages.yaml:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 |
--- kubespray: common: - socat - e2fsprogs - xfsprogs - ebtables - bash-completion - ipvsadm - ipset - conntrack yum: - nss - device-mapper-libs - conntrack-tools apt: - python-apt - python3-apt - aufs-tools - apt-transport-https - software-properties-common docker: - docker-ce-20.10.17 - docker-ce-cli-20.10.17 - containerd.io-1.6.7 middleware: yum: - keepalived - ibus - gcc - gcc-c++ - pcre - pcre-devel - libtool - libtool-ltdl-devel - libevent-devel - perl - perl-devel - zlib - zlib-devel - expat - gd - gd-devel - make - lua - ncurses-devel - flex - libatomic - iproute - procps-ng - fontconfig - bind-utils - libffi-devel - readline - readline-devel - tcl - cifs-utils - openssl - openssl-devel - openssl-libs - chrony - libselinux-python - python3-libselinux common: - curl - jq - bc - tar - telnet - expect - xz - unzip - zip - nc - lrzsz - lsof - lvm2 - sshpass - vim - wget - ethtool - net-tools - rsync - dos2unix - ntp - ntpdate yum: - nfs-utils - createrepo - nc - httpd-tools apt: - nfs-common - apt-transport-https - ca-certificates - gnupg - lsb-release - aptitude - dpkg-dev - gnupg2 - netcat - apache2-utils - libgd-dev - expat - gcc - make - libpcre3 - libpcre3-dev - zlib1g - zlib1g-dev - openssl - libssl-dev - perl centos7: - libselinux-python - s3fs-fuse kylin10: - compat-openssl10 uos: - compat-openssl10 openeuler: nfs: - compat-openssl10 |
定义各系统构建使用的Dockerfile 下面是制作麒麟离线源的Dockerfile:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
ARG FROM="dockerhub.xxx.com/os/kylin:v10-sp3" FROM $FROM as kylin ARG OS_VERSION=10 ARG BUILD_TOOLS="curl dnf-utils createrepo" ARG BASE_URL=http://xxx.oss-cn-beijing.aliyuncs.com/download WORKDIR /kylin/$OS_VERSION/os COPY packages.yaml . RUN set -x && ARCH=$(uname -m) \ && curl -o /usr/bin/yq $BASE_URL/binary/yq-linux-$ARCH && chmod +x /usr/bin/yq \ && yum install -q -y ${BUILD_TOOLS} || true \ \ && yq eval '.common[],.yum[],.kylin10[],.kubespray.common[],.kubespray.yum[],.middleware.yum[]' packages.yaml > packages.list \ && sort -u packages.list | xargs repotrack -d 9 --destdir "${ARCH}/Packages" RUN set -x && ARCH=$(uname -m) \ && if [ "$ARCH" == 'x86_64' ]; then centos_repo='Centos-7.repo'; else centos_repo='Centos-altarch-7.repo'; fi \ # install docker && curl -o /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo \ && sed -i -e 's/$releasever/7/g' -e 's/https/http/g' /etc/yum.repos.d/docker-ce.repo \ && curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/${centos_repo} \ && sed -i -e 's/$releasever/7/g' -e 's/https/http/g' /etc/yum.repos.d/CentOS-Base.repo \ && yum makecache \ && yq eval '.docker[]' packages.yaml | xargs repotrack -d 9 --destdir "${ARCH}/Packages" \ \ && rm -rf packages.yaml packages.list \ && createrepo -d "${ARCH}" FROM scratch COPY --from=kylin /kylin ./kylin |
中科方德在适配过程中出现了很多问题，总结如下： 官方没有docker-ce-20.10.17版本，需要配置docker el8源 官方没有sshpass等包，需要配置centos7的源 Read more…