#!/usr/bin/python
# -*- coding: utf-8 -*-
#import commands
import jinja2
import yaml
import sys
import os
import re
#import traceback
import socket
import subprocess
import time
import json
import argparse
import logging
mdhms=time.strftime('%m-%d-%H-%M',time.localtime(time.time()))
logdir='./log'
if not os.path.exists(logdir):
os.makedirs(logdir)
logfile=os.path.join(logdir, 'build_'+mdhms+'.log')
logfile_latest=os.path.join(logdir, 'latest.log')
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
BASIC_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = '%m/%d/%Y %H:%M:%S'
formatter = logging.Formatter(BASIC_FORMAT, DATE_FORMAT)
# StreamHandler
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
# stream_handler.setLevel(logging.INFO)
# FileHandler
file_handler = logging.FileHandler(logfile)
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.INFO)
logger.addHandler(stream_handler)
logger.addHandler(file_handler)
if os.path.exists('/root/.python_env/lib/python2.7/site-packages/'):
sys.path.insert(0, "/root/.python_env/lib/python2.7/site-packages/")
try:
import docker
except:
print ("installing docker-py module, please wait a second...")
sys.stdout.flush()
os.system("pip install docker")
import docker
# reload(sys)
# sys.setdefaultencoding('utf8')
current_dir = os.getcwd()
download_dir = os.path.join(current_dir,'download')
download_http_port = "8899"
dockerfile_dir = os.path.join(current_dir, 'dockerfile')
if not os.path.exists(dockerfile_dir):
os.makedirs(dockerfile_dir)
#docker_client = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=5)
#docker_client = docker.from_env()
def get_host_ip():
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
finally:
s.close()
return ip
def start_download_http_server():
check_http_process="ps -efa|grep SimpleHTTPServerWithUpload.py|grep -v grep"
check_http_port="netstat -antp | grep %s | grep LISTEN >/dev/null 2>&1" % (download_http_port)
start_server="nohup python SimpleHTTPServerWithUpload.py %s > /dev/null 2>&1 &" % (download_http_port)
http_process = subprocess.call(check_http_process, shell=True)
http_port = subprocess.call(check_http_port, shell=True)
if http_process != 0 and http_port != 0:
os.chdir(download_dir)
subprocess.call(start_server, shell=True)
os.chdir(current_dir)
def stop_download_http_server():
kill_http_server = "kill -9 `ps -efa|grep SimpleHTTPServerWithUpload.py|grep -v grep|awk '{print $2}'`"
subprocess.call(kill_http_server, shell=True)
def render_dockerfile(item):
dockerfile = os.path.join(dockerfile_dir, 'Dockerfile.' + item['arch'] + '-' + item['image'].replace(':', '-'))
profile = template_env.get_template(item['template'])
content = profile.render(item)
with open(dockerfile, 'w') as fp:
fp.write(content)
return dockerfile
class Docker_Client():
def __init__(self):
# self.client = docker.DockerClient(version='auto', base_url='unix://var/run/docker.sock')
self.client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
#self.username = username
#self.password = password
def build_image(self, path, dockerfile, image, nocache=True, network_mode="host", memory_limit="3MB", buildargs=""):
limits = {
# Always disable memory swap for building, since mostly
# nothing good can come of that.
'memswap': -1
}
if memory_limit:
limits['memory'] = memory_limit
#client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
logger.info("--> Building %s" % image)
try:
build = self.client.build(
path=path,
dockerfile=dockerfile,
tag=image,
buildargs=buildargs,
decode=True,
forcerm=True,
container_limits=limits,
nocache=nocache,
network_mode=network_mode,
encoding="utf-8"
)
for line in build:
logger.debug([line['stream'] if 'stream' in line else line][0])
except Exception as e:
logger.error("build %s failed: %s" % (image, e), exc_info=True)
sys.exit(1)
def push_image(self, image, full_image, registry, username="admin", password="Harbor12345"):
# tag image
logger.info("--> tag %s to %s" % (image, full_image))
self.client.tag(image, full_image, force=True)
try:
# login registry
self.client.login(username=username, password=password, email=None, registry=registry)
# Build a progress setup for each layer, and only emit per-layer
# info every 1.5s
# layers = {}
last_emit_time = time.time()
for line in self.client.push(full_image, stream=True, decode=True):
# progress = json.loads(line.decode('utf-8'))
# # print '-'*5, progress
# if 'error' in progress:
# logger.error(progress['error'], extra=dict(phase='failed'))
# sys.exit(1)
# if 'id' not in progress:
# continue
# if 'progressDetail' in progress and progress['progressDetail']:
# layers[progress['id']] = progress['progressDetail']
# else:
# layers[progress['id']] = progress['status']
# logger.debug('pushing %s' % layers)
# if time.time() - last_emit_time > 1.5:
# print 'pushing %s' % layers
# last_emit_time = time.time()
if time.time() - last_emit_time > 1.5:
logger.debug(line)
last_emit_time = time.time()
logger.info('--> Pushed %s ' % full_image)
except Exception as e:
logger.error("push_image %s failed: %s" % (full_image, e), exc_info=True)
sys.exit(1)
if __name__ == '__main__':
try:
parser = argparse.ArgumentParser(description = 'build')
parser.add_argument ('--config', default = "./config/config.yml", type=str, help = 'please input config file.')
parser.add_argument ('--enable-http-server', default = "true", type=str, help = 'auto start a http server for image build resources. default true')
global args
args = parser.parse_args()
if args.enable_http_server == "true":
start_download_http_server()
template_loader = jinja2.FileSystemLoader(searchpath="./templates")
template_env = jinja2.Environment(loader=template_loader)
template_env.trim_blocks = True
config_file = open(args.config)
data_config = {}
data_config.update(yaml.safe_load(config_file))
download_http_ip = get_host_ip()
base_url = 'http://' + download_http_ip + ':' + download_http_port
data = {}
data.update(data_config)
docker_client = Docker_Client()
for item in data['images']:
item['base_url'] = base_url
image = item['image']
# image_tag = image.split(':')[1]
dockerfile = render_dockerfile(item)
logger.info('-'*20 + image + '-'*20)
docker_client.build_image(dockerfile_dir, dockerfile, image)
for registry in data['registry']:
host = registry['host']
username = registry['username']
password = registry['password']
repository = registry['repository']
full_image = os.path.join(host, repository, image)
docker_client.push_image(image, full_image, host, username, password)
logger.info('-'*60 + '\n\n')
#print("build image: " + full_image)
#sys.stdout.flush()
# if os.system("docker build --network=host -f %s %s -t %s" % (dockerfile, dockerfile_dir, target_image)) != 0:
# print("image %s build failed." % (full_image))
# sys.stdout.flush()
# sys.exit(1)
# os.system("docker tag %s %s" % (target_image, full_image))
# os.system("docker login %s --username %s --password %s" % (data['registry']['host'], data['registry']['username'], data['registry']['password']))
# if os.system("docker push %s" % (full_image)) != 0:
# print("image %s push failed." % (full_image))
# sys.stdout.flush()
# sys.exit(1)
os.system("ln -sf %s %s" % (os.path.abspath(logfile), os.path.abspath(logfile_latest)))
if args.enable_http_server == "true":
stop_download_http_server()
except Exception as e:
logger.error('Faild Info: %s' % e, exc_info=True)
if args.enable_http_server == "true":
stop_download_http_server()
sys.exit(1)