sean – Page 18 – 一江春水向东流

python3爬虫学习笔记之urllib库的使用

python3爬虫学习笔记之urllib库的使用基本使用

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request

request_url = 'http://www.baidu.com'           # 需要请求的URL地址
response = urllib.request.urlopen(request_url) # 发起请求
print(response.read().decode('utf-8'))         # 打印响应的文本，并进行UTF-8

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request

request_url = 'http://www.baidu.com' # 需要请求的URL地址

response = urllib.request.urlopen(request_url) # 发起请求

print(response.read().decode('utf-8')) # 打印响应的文本，并进行UTF-8

urlopen返回对象提供方法： read(), readline(), readlines(), fileno(), close()：对HTTPResponse类型数据进行操作 info()：返回HTTPMessage对象，表示远程服务器返回的头信息 getcode()：返回Http状态码。如果是http请求，200请求成功完成、404网址未找到等等 geturl()：返回请求的url 请求的数据传送 GET数据

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request
import urllib.parse

get_data = {'username': 'aaa', 'password': 'bbb'}          # 此处将GET的数据定义为一个字典
get_data_encode = urllib.parse.urlencode(get_data)         # 将GET的数据进行编码

request_url = 'https://www.zhihu.com/#signin'              # 需要请求的URL地址
request_url += '?' + get_data_encode                       # 追加GET参数到URL后面

# https://www.zhihu.com/#signin?username=aaa&password=bbb
print(request_url)

# 发起请求
response = urllib.request.urlopen(request_url)
print(response.read().decode('utf-8'))         # 打印响应的文本，并进行UTF-8解码

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request

import urllib.parse

get_data = {'username': 'aaa', 'password': 'bbb'} # 此处将GET的数据定义为一个字典

get_data_encode = urllib.parse.urlencode(get_data) # 将GET的数据进行编码

request_url = 'https://www.zhihu.com/#signin' # 需要请求的URL地址

request_url += '?' + get_data_encode # 追加GET参数到URL后面

# https://www.zhihu.com/#signin?username=aaa&password=bbb

print(request_url)

# 发起请求

response = urllib.request.urlopen(request_url)

print(response.read().decode('utf-8')) # 打印响应的文本，并进行UTF-8解码

POST数据

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request
import urllib.parse

post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'}      # 此处将POST的数据定义为一个字典
post_data_encode = urllib.parse.urlencode(post_data)        # 将POST的数据进行编码

# UTF-8编码
# 否则会报错：POST data should be bytes or an iterable of bytes. It cannot be of type str.
post_data_encode = post_data_encode.encode(encoding='utf-8')
request_url = 'http://www.lagou.com/jobs/positionAjax.json?'               # 需要请求的URL地址

# 发起请求
# 此处增加了第二个参数为传送的POST数据（默认为None）
# 第三个参数为请求超时时间，默认为socket._GLOBAL_DEFAULT_TIMEOUT
response = urllib.request.urlopen(request_url, post_data_encode, 3)
print(response.read().decode('utf-8'))         # 打印响应的文本，并进行UTF-8解码

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request

import urllib.parse

post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'} # 此处将POST的数据定义为一个字典

post_data_encode = urllib.parse.urlencode(post_data) # 将POST的数据进行编码

# UTF-8编码

# 否则会报错：POST data should be bytes or an iterable of bytes. It cannot be of type str.

post_data_encode = post_data_encode.encode(encoding='utf-8')

request_url = 'http://www.lagou.com/jobs/positionAjax.json?' # 需要请求的URL地址

# 发起请求

# 此处增加了第二个参数为传送的POST数据（默认为None）

# 第三个参数为请求超时时间，默认为socket._GLOBAL_DEFAULT_TIMEOUT

response = urllib.request.urlopen(request_url, post_data_encode, 3)

print(response.read().decode('utf-8')) # 打印响应的文本，并进行UTF-8解码

使用Request &&　设置Headers属性很多网站对非浏览器的访问都做了限制，所以如果我们要完全模拟浏览器去访问网站的话，必须要先设置Headers的属性使用chrome浏览器按F12，点击访问的链接，Headers -> Request Headers User-Agent:Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 如果服务器对页面访问来源做了限制，则需要设置Headers的Referer属性

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request
import urllib.parse

user_agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87'
referer = 'http://www.lagou.com/jobs/positionAjax.json?'
post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'}                              # 此处将POST的数据定义为一个字典
headers = {'User-Agent': user_agent, 'Referer': referer}                            # Headers属性初始化
post_data_encode = urllib.parse.urlencode(post_data)                                # 将POST的数据进行编码

# UTF-8编码
# 否则会报错：POST data should be bytes or an iterable of bytes. It cannot be of type str.
post_data_encode = post_data_encode.encode(encoding='utf-8')
request_url = 'http://www.lagou.com/zhaopin/Python/?labelWords=label'               # 需要请求的URL地址

# 使用Request来设置Headers
request = urllib.request.Request(request_url, post_data_encode, headers)

response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))         # 打印响应的文本，并进行UTF-8解码

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

import urllib.request

import urllib.parse

user_agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87'

referer = 'http://www.lagou.com/jobs/positionAjax.json?'

post_data = {'first': 'true', 'pn': 1, 'kd': 'Python'} # 此处将POST的数据定义为一个字典

headers = {'User-Agent': user_agent, 'Referer': referer} # Headers属性初始化

post_data_encode = urllib.parse.urlencode(post_data) # 将POST的数据进行编码

# UTF-8编码

# 否则会报错：POST data should be bytes or an iterable of bytes. It cannot be of type str.

post_data_encode = post_data_encode.encode(encoding='utf-8')

request_url = 'http://www.lagou.com/zhaopin/Python/?labelWords=label' # 需要请求的URL地址

# 使用Request来设置Headers

request = urllib.request.Request(request_url, post_data_encode, headers)

response = urllib.request.urlopen(request)

print(response.read().decode('utf-8')) # 打印响应的文本，并进行UTF-8解码

Proxy（代理）的设置如果网站限制了IP访问的次数，则需要更换代理服务器，以免被禁止访问

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

from urllib import request

request_url = 'http://www.lagou.com/jobs/positionAjax.json?'
proxy = request.ProxyHandler({'http': '61.136.115.147:3128'})   # 设置代理服务器
opener = request.build_opener(proxy)                            # 挂载opener
request.install_opener(opener)                                  # 安装opener
response = request.urlopen(request_url)
print(response.read().decode('utf-8'))         # 打印响应的文本，并进行UTF-8解码

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

from urllib import request

request_url = 'http://www.lagou.com/jobs/positionAjax.json?'

proxy = request.ProxyHandler({'http': '61.136.115.147:3128'}) # 设置代理服务器

opener = request.build_opener(proxy) # 挂载opener

request.install_opener(opener) # 安装opener

response = request.urlopen(request_url)

print(response.read().decode('utf-8')) # 打印响应的文本，并进行UTF-8解码

异常处理

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

from urllib import request
from urllib import error        # 引入错误处理的包

request_url = 'http://www.lagou.com/jobs/positionAjax.json?'
proxy = request.ProxyHandler({'http': '116.204.1.111:8081'}) # 设置代理服务器
opener = request.build_opener(proxy)                         # 挂载opener
request.install_opener(opener)                               # 安装opener
try:
    response = request.urlopen(request_url)
except error.HTTPError as e:
    print(e.code)                   # 打印错误码
    print(e.msg)                    # 打印错误信息

#!/usr/bin/env python

# -*- coding: utf-8 -*-

__author__ = 'JustFantasy'

from urllib import request

from urllib import error # 引入错误处理的包

request_url = 'http://www.lagou.com/jobs/positionAjax.json?'

proxy = request.ProxyHandler({'http': '116.204.1.111:8081'}) # 设置代理服务器

opener = request.build_opener(proxy) # 挂载opener

request.install_opener(opener) # 安装opener

try:

response = request.urlopen(request_url)

except error.HTTPError as e:

print(e.code) # 打印错误码

print(e.msg) # 打印错误信息

By sean, 8 years10/08/2017 ago

未分类

Python虚拟环境设置

Python虚拟环境设置 2018-4-2更新： centos7创建虚拟环境在最新的CentOS版本7.3中，自带的Python版本依旧是2.7，而Django的建议版本已经到了3.x。所以需要安装新版本的Python，同时保留旧的Python环境。首先安装epel源 1 yum install epel–release 查看可用的python3版本 1 yum list python3* 可以看见列表里提供Python 3.4 1 yum –y install python34 下面安装pip，python2.7和pyhon3.4的pip是可以同时存在的，默认分别为pip，pip3 1 2 yum install python–pip # 安装python2.7的pip yum install python34–pip # 安装python3.4的pip，两者可以同时存在查看pip版本信息 1 2 3 4 ~# pip -V pip 8.1.2 from /usr/lib/python2.7/site–packages (python 2.7) ~# pip3 Read more…

By sean, 8 years10/06/2017 ago

未分类

nginx日志切割脚本

# -*- coding: utf-8 -*-
#!/usr/bin/env python
# author:sean
# nginx 日志切割脚本
# www.a.com.log  www.a.com-YesterDay(%Y-%m-%d).log

import os
import datetime

LogPath = "/data/logs/nginx" #日志所在目录
BakDir = "/data/logs/nginx/bak" #要备份到的目录
YesterDay = (datetime.datetime.now()+datetime.timedelta(days=-1)).strftime("%Y-%m-%d")
if os.path.exists(BakDir):
    pass
else:
    os.mkdir(BakDir)

for root,dirs,files in os.walk(LogPath):
    for f in files:
        filename,ext = os.path.splitext(f)
        os.rename("{0}/{1}".format(root,f),"{0}/{1}-{2}{3}".format(root,filename,YesterDay,ext))

os.system("kill -USR1 'cat /usr/local/openresty/nginx/logs/nginx.pid'")
time.sleep(5)
os.system('find {0} -type f -name "*.log" -mtime +7 -exec rm -rf {} \;'.foramt(BakDir))

# -*- coding: utf-8 -*-

#!/usr/bin/env python

# author:sean

# nginx 日志切割脚本

# www.a.com.log www.a.com-YesterDay(%Y-%m-%d).log

import os

import datetime

LogPath = "/data/logs/nginx" #日志所在目录

BakDir = "/data/logs/nginx/bak" #要备份到的目录

YesterDay = (datetime.datetime.now()+datetime.timedelta(days=-1)).strftime("%Y-%m-%d")

if os.path.exists(BakDir):

pass

else:

os.mkdir(BakDir)

for root,dirs,files in os.walk(LogPath):

for f in files:

filename,ext = os.path.splitext(f)

os.rename("{0}/{1}".format(root,f),"{0}/{1}-{2}{3}".format(root,filename,YesterDay,ext))

os.system("kill -USR1 'cat /usr/local/openresty/nginx/logs/nginx.pid'")

time.sleep(5)

os.system('find {0} -type f -name "*.log" -mtime +7 -exec rm -rf {} \;'.foramt(BakDir))

By sean, 8 years10/05/2017 ago

未分类

shadowSocks 从gwflist更新PAC脚本

shadowsocks从gfw更新pac时更新失败, 下面脚本可以代替更新功能, 适用于mac系统:

#!/bin/bash
# update_gfwlist.sh
# Author : VincentSit
# Copyright (c) http://xuexuefeng.com
#
# Example usage
#
# ./whatever-you-name-this.sh
#
# Task Scheduling (Optional)
#
#	crontab -e
#
# add:
# 30 9 * * * sh /path/whatever-you-name-this.sh
#
# Now it will update the PAC at 9:30 every day.
#
# Remember to chmod +x the script.


GFWLIST="https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt"
PROXY="127.0.0.1:1080"
USER_RULE_NAME="user-rule.txt"

check_module_installed()
{
	pip list | grep gfwlist2pac &> /dev/null

	if [ $? -eq 1 ]; then
		echo "Installing gfwlist2pac."

		pip install gfwlist2pac
	fi
}

update_gfwlist()
{
	echo "Downloading gfwlist."

	curl -s "$GFWLIST" --fail --socks5-hostname "$PROXY" --output /tmp/gfwlist.txt

	if [[ $? -ne 0 ]]; then
		echo "abort due to error occurred."
    exit 1
	fi

	cd ~/.ShadowsocksX || exit 1

	if [ -f "gfwlist.js" ]; then
		mv gfwlist.js ~/.Trash
	fi

	if [ ! -f $USER_RULE_NAME ]; then
		touch $USER_RULE_NAME
	fi

	/usr/local/bin/gfwlist2pac \
    --input /tmp/gfwlist.txt \
    --file gfwlist.js \
    --proxy "SOCKS5 $PROXY; SOCKS $PROXY; DIRECT" \
    --user-rule $USER_RULE_NAME \
    --precise

  rm -f /tmp/gfwlist.txt

  echo "Updated."
}

check_module_installed
update_gfwlist

#!/bin/bash

# update_gfwlist.sh

# Author : VincentSit

# Copyright (c) http://xuexuefeng.com

# Example usage

# ./whatever-you-name-this.sh

# Task Scheduling (Optional)

# crontab -e

# add:

# 30 9 * * * sh /path/whatever-you-name-this.sh

# Now it will update the PAC at 9:30 every day.

# Remember to chmod +x the script.

GFWLIST="https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt"

PROXY="127.0.0.1:1080"

USER_RULE_NAME="user-rule.txt"

check_module_installed()

{

pip list | grep gfwlist2pac &> /dev/null

if [ $? -eq 1 ]; then

echo "Installing gfwlist2pac."

pip install gfwlist2pac

}

update_gfwlist()

{

echo "Downloading gfwlist."

curl -s "$GFWLIST" --fail --socks5-hostname "$PROXY" --output /tmp/gfwlist.txt

if [[ $? -ne 0 ]]; then

echo "abort due to error occurred."

exit 1

cd ~/.ShadowsocksX || exit 1

if [ -f "gfwlist.js" ]; then

mv gfwlist.js ~/.Trash

if [ ! -f $USER_RULE_NAME ]; then

touch $USER_RULE_NAME

/usr/local/bin/gfwlist2pac \

--input /tmp/gfwlist.txt \

--file gfwlist.js \

--proxy "SOCKS5 $PROXY; SOCKS $PROXY; DIRECT" \

--user-rule $USER_RULE_NAME \

--precise

rm -f /tmp/gfwlist.txt

echo "Updated."

}

check_module_installed

update_gfwlist

转自: https://gist.github.com/VincentSit/b5b112d273513f153caf23a9da112b3a

By sean, 8 years10/05/2017 ago

未分类

实现Openwrt路由器智能翻墙

openwrt固件实现智能翻墙国庆放假前几天一直在家折腾路由器自动翻墙, 固件刷了好几遍, 虽然最后也没成功, 还是记录下配置过程, 以后再想做时用得着. 我的路由器是Netgear R6100, 去 https://wiki.openwrt.org/toh/hwdata/netgear/netgear_r6100 查到路由器信息, 安装安装软件有三种方法, 一, 通过配置源在线安装; 二, 通过脚本安装. 三, 通过下载安装包安装; 一, 最简单的就是通过源安装: 由于openwrt-dist.sourceforge.net源不翻墙的话不能用, 会有bad address的错误提示, 实际配置需要配置反代源, 参考了 http://openwrt-dist.colendres.com/ , 最后配置如下: First, add openwrt-dist.pub file into opkg’s keys:

wget http://openwrt-dist.sourceforge.net/packages/openwrt-dist.pub
opkg-key add openwrt-dist.pub

1 2	wget http://openwrt-dist.sourceforge.net/packages/openwrt-dist.pub opkg-key add openwrt-dist.pub

添加如下到/etc/opkg.conf, 注意将签名验证给注释掉

src/gz openwrt_dist http://openwrt-dist.colendres.com/packages/OpenWrt/base/ar71xx
#或src/gz openwrt_dist http://openwrt-dist.colendres.com/packages/OpenWrt/base/{architecture}
src/gz openwrt_dist_luci http://openwrt-dist.colendres.com/packages/OpenWrt/luci

src/gz openwrt_dist http://openwrt-dist.colendres.com/packages/OpenWrt/base/ar71xx

#或src/gz openwrt_dist http://openwrt-dist.colendres.com/packages/OpenWrt/base/{architecture}

src/gz openwrt_dist_luci http://openwrt-dist.colendres.com/packages/OpenWrt/luci

然后安装相应软件:

opkg update
opkg install iptables-mod-nat-extra iptables-mod-tproxy ipset
opkg remove dnsmasq && opkg install dnsmasq-full
opkg install ChinaDNS
opkg install luci-app-chinadns
opkg install dns-forwarder
opkg install luci-app-dns-forwarder
opkg install shadowsocks-libev
opkg install luci-app-shadowsocks
opkg install ShadowVPN
opkg install luci-app-shadowvpn

opkg update

opkg install iptables-mod-nat-extra iptables-mod-tproxy ipset

opkg remove dnsmasq && opkg install dnsmasq-full

opkg install ChinaDNS

opkg install luci-app-chinadns

opkg install dns-forwarder

opkg install luci-app-dns-forwarder

opkg install shadowsocks-libev

opkg install luci-app-shadowsocks

opkg install ShadowVPN

opkg install luci-app-shadowvpn

注: 每次路由器重启后, 都要先opkg update下才能安装软件二, 通过脚本安装

wget http://openwrt-dist.colendres.com/auto_install_openwrt.sh
chmod +x auto_install_openwrt.sh
./auto_install_openwrt.sh

wget http://openwrt-dist.colendres.com/auto_install_openwrt.sh

chmod +x auto_install_openwrt.sh

./auto_install_openwrt.sh

三, 下载所有安装包然后opkg install 安装 Read more…

By sean, 8 years10/03/2017 ago

未分类

centos系统优化脚本

#!/bin/bash
#written by mofansheng@2015-11-03
#system optimization script
#The fllow apply to CentOS 6.x
. /etc/init.d/functions
 
function check_ok(){
  if [ $? -eq 0 ]
   then
     echo ""
     continue
  else
     echo "pls check error"
     exit
  fi
}
 
cat<<EOF
-----------------------------------------------------------------------
|                     system optimization                             |
-----------------------------------------------------------------------
EOF
#1. 更新系统
yum update -y
yum install net-tools wget telnet perl perl-devel kernel-devel bash-completion ntpdate vim
#yum install gcc cmake bzip2-devel curl-devel db4-devel libjpeg-devel libpng-devel freetype-devel libXpm-devel gmp-devel libc-client-devel openldap-devel unixODBC-devel postgresql-devel sqlite-devel aspell-devel net-snmp-devel libxslt-devel libxml2-devel pcre-devel pspell-devel libmemcached libmemcached-devel zlib-devel  vim wget   lrzsz  tree
yum groupinstall "Development tools" -y
#2. 给/etc/rc.local添加执行权限
chmod +x /etc/rc.d/rc.local

#3. 关闭不必要服务close unimportant system services
echo "===Close unimportant system services,it will take serval mintinues==="
for i in `systemctl list-unit-files |grep 'enabled'| awk '{print $1}' |grep -Ev "crond|sshd|sysstat|rsyslog|network|dbus|systemd|multi|default|NetworkManager|auditd"`; do 
	systemctl disable $i;systemctl stop $i;systemctl status $i
done
check_ok
action "Close unimportant system services" /bin/true
 
 
#4. close selinux
echo "===close SELINUX==="
if [ `getenforce` != "Disabled" ]
then
 sed -i  '/^SELINUX/s/enforcing/disabled/g' /etc/selinux/config
 setenforce 0
 echo "selinux is disabled"
else
 action "SELINUX is closed" /bin/true
fi
check_ok
action "Close SELINUX" /bin/true
 
#5. 关闭防火墙安装iptables
systemctl   stop   firewalld.service 
systemctl   disable firewalld.service
yum install iptables-services -y

#6. 修改主机名
hostnamectl set-hostname vm1

#7. 设置字符集 
yum -y install kde-l10n-Chinese
localectl set-locale LANG=zh_CN.utf8
echo "LANG=zh_CN.UTF-8" > /etc/environment
echo "LC_ALL=zh_CN.UTF-8" >> /etc/environment
echo "LANG=zh_CN.utf8" > /etc/locale.conf
source  /etc/locale.conf

#8. 加大打开文件数的限制（open files）和内核优化
echo "* soft nofile 65535" >> /etc/security/limits.conf 
echo "* hard nofile 65535" >> /etc/security/limits.conf
echo "* soft nproc 65535" >> /etc/security/limits.conf
echo "* hard nproc 65535" >> /etc/security/limits.conf

cat >> /etc/sysctl.conf << "EOF"
#CTCDN系统优化参数
fs.file-max = 655536
#关闭ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
#决定检查过期多久邻居条目
net.ipv4.neigh.default.gc_stale_time=120
#使用arp_announce / arp_ignore解决ARP映射问题
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.lo.arp_announce=2
# 避免放大攻击
net.ipv4.icmp_echo_ignore_broadcasts = 1
# 开启恶意icmp错误消息保护
net.ipv4.icmp_ignore_bogus_error_responses = 1
#关闭路由转发
net.ipv4.ip_forward = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
#开启反向路径过滤
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
#处理无源路由的包
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
#关闭sysrq功能
kernel.sysrq = 0
#core文件名中添加pid作为扩展名
kernel.core_uses_pid = 1
# 开启SYN洪水攻击保护
net.ipv4.tcp_syncookies = 1
#修改消息队列长度
kernel.msgmnb = 65536
kernel.msgmax = 65536
#设置最大内存共享段大小bytes
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
#timewait的数量，默认180000
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_wmem = 4096        16384   4194304
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
#每个网络接口接收数据包的速率比内核处理这些包的速率快时，允许送到队列的数据包的最大数目
net.core.netdev_max_backlog = 262144
#限制仅仅是为了防止简单的DoS 攻击
net.ipv4.tcp_max_orphans = 3276800
#未收到客户端确认信息的连接请求的最大值
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0
#内核放弃建立连接之前发送SYNACK 包的数量
net.ipv4.tcp_synack_retries = 1
#内核放弃建立连接之前发送SYN 包的数量
net.ipv4.tcp_syn_retries = 1
#启用timewait 快速回收
net.ipv4.tcp_tw_recycle = 1
#开启重用。允许将TIME-WAIT sockets 重新用于新的TCP 连接
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
#当keepalive 起用的时候，TCP 发送keepalive 消息的频度。缺省是2 小时
net.ipv4.tcp_keepalive_time = 1800
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl = 15
#允许系统打开的端口范围
net.ipv4.ip_local_port_range = 1024    65000
#修改防火墙表大小，默认65536
net.netfilter.nf_conntrack_max=655350
net.netfilter.nf_conntrack_tcp_timeout_established=1200
# 确保无人能修改路由表
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
EOF
#生效
sysctl -p
#用户进程限制
sed -i 's#4096#65535#g' /etc/security/limits.d/20-nproc.conf  #加大普通用户限制  也可以改为unlimited
 
#set SSH
sed -i 's/#UseDNS yes/UseDNS no/g' /etc/ssh/sshd_config
sed -i 's/#Port 22/Port 65500/g' /etc/ssh/sshd_config
service sshd restart
 
 
#set system files permission
chmod 600 /etc/passwd
chmod 600 /etc/group
chmod 600 /etc/shadow
chmod 600 /etc/gshadow
 
 
#set ntp
yum install ntpdate -y
ntpdate ntp.fudan.edu.cn
echo "* 3 * * * /usr/sbin/ntpdate ntp.fudan.edu.cn >/dev/null 2>&1" >>/etc/crontab
service crond restart
check_ok
action "ntpdate is installed and add in crontab" /bin/true
 
 
#set vim
echo "===install vim,it will take serval mintinues==="
yum install vim-enhanced -y &>/dev/null
alias vi=vim
echo "alias vi=vim" >>/root/.bashrc
check_ok
action "vim is installed" /bin/true
 
 
#set yum repos
echo "===update yum repos,it will take serval mintinues==="
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all &>/dev/null
yum makecache &>/dev/null
check_ok
action  "yum repos update is ok" /bin/true

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

#!/bin/bash

#written by mofansheng@2015-11-03

#system optimization script

#The fllow apply to CentOS 6.x

. /etc/init.d/functions

function check_ok(){

if [ $? -eq 0 ]

then

echo ""

continue

else

echo "pls check error"

exit

}

cat<<EOF

-----------------------------------------------------------------------

| system optimization |

-----------------------------------------------------------------------

EOF

#1. 更新系统

yum update -y

yum install net-tools wget telnet perl perl-devel kernel-devel bash-completion ntpdate vim

#yum install gcc cmake bzip2-devel curl-devel db4-devel libjpeg-devel libpng-devel freetype-devel libXpm-devel gmp-devel libc-client-devel openldap-devel unixODBC-devel postgresql-devel sqlite-devel aspell-devel net-snmp-devel libxslt-devel libxml2-devel pcre-devel pspell-devel libmemcached libmemcached-devel zlib-devel vim wget lrzsz tree

yum groupinstall "Development tools" -y

#2. 给/etc/rc.local添加执行权限

chmod +x /etc/rc.d/rc.local

#3. 关闭不必要服务close unimportant system services

echo "===Close unimportant system services,it will take serval mintinues==="

for i in `systemctl list-unit-files |grep 'enabled'| awk '{print $1}' |grep -Ev "crond|sshd|sysstat|rsyslog|network|dbus|systemd|multi|default|NetworkManager|auditd"`; do

systemctl disable $i;systemctl stop $i;systemctl status $i

done

check_ok

action "Close unimportant system services" /bin/true

#4. close selinux

echo "===close SELINUX==="

if [ `getenforce` != "Disabled" ]

then

sed -i '/^SELINUX/s/enforcing/disabled/g' /etc/selinux/config

setenforce 0

echo "selinux is disabled"

else

action "SELINUX is closed" /bin/true

check_ok

action "Close SELINUX" /bin/true

#5. 关闭防火墙安装iptables

systemctl stop firewalld.service

systemctl disable firewalld.service

yum install iptables-services -y

#6. 修改主机名

hostnamectl set-hostname vm1

#7. 设置字符集

yum -y install kde-l10n-Chinese

localectl set-locale LANG=zh_CN.utf8

echo "LANG=zh_CN.UTF-8" > /etc/environment

echo "LC_ALL=zh_CN.UTF-8" >> /etc/environment

echo "LANG=zh_CN.utf8" > /etc/locale.conf

source /etc/locale.conf

#8. 加大打开文件数的限制（open files）和内核优化

echo "* soft nofile 65535" >> /etc/security/limits.conf

echo "* hard nofile 65535" >> /etc/security/limits.conf

echo "* soft nproc 65535" >> /etc/security/limits.conf

echo "* hard nproc 65535" >> /etc/security/limits.conf

cat >> /etc/sysctl.conf << "EOF"

#CTCDN系统优化参数

fs.file-max = 655536

#关闭ipv6

net.ipv6.conf.all.disable_ipv6 = 1

net.ipv6.conf.default.disable_ipv6 = 1

#决定检查过期多久邻居条目

net.ipv4.neigh.default.gc_stale_time=120

#使用arp_announce / arp_ignore解决ARP映射问题

net.ipv4.conf.default.arp_announce = 2

net.ipv4.conf.all.arp_announce=2

net.ipv4.conf.lo.arp_announce=2

# 避免放大攻击

net.ipv4.icmp_echo_ignore_broadcasts = 1

# 开启恶意icmp错误消息保护

net.ipv4.icmp_ignore_bogus_error_responses = 1

#关闭路由转发

net.ipv4.ip_forward = 0

net.ipv4.conf.all.send_redirects = 0

net.ipv4.conf.default.send_redirects = 0

#开启反向路径过滤

net.ipv4.conf.all.rp_filter = 1

net.ipv4.conf.default.rp_filter = 1

#处理无源路由的包

net.ipv4.conf.all.accept_source_route = 0

net.ipv4.conf.default.accept_source_route = 0

#关闭sysrq功能

kernel.sysrq = 0

#core文件名中添加pid作为扩展名

kernel.core_uses_pid = 1

# 开启SYN洪水攻击保护

net.ipv4.tcp_syncookies = 1

#修改消息队列长度

kernel.msgmnb = 65536

kernel.msgmax = 65536

#设置最大内存共享段大小bytes

kernel.shmmax = 68719476736

kernel.shmall = 4294967296

#timewait的数量，默认180000

net.ipv4.tcp_max_tw_buckets = 6000

net.ipv4.tcp_sack = 1

net.ipv4.tcp_window_scaling = 1

net.ipv4.tcp_rmem = 4096 87380 4194304

net.ipv4.tcp_wmem = 4096 16384 4194304

net.core.wmem_default = 8388608

net.core.rmem_default = 8388608

net.core.rmem_max = 16777216

net.core.wmem_max = 16777216

#每个网络接口接收数据包的速率比内核处理这些包的速率快时，允许送到队列的数据包的最大数目

net.core.netdev_max_backlog = 262144

#限制仅仅是为了防止简单的DoS 攻击

net.ipv4.tcp_max_orphans = 3276800

#未收到客户端确认信息的连接请求的最大值

net.ipv4.tcp_max_syn_backlog = 262144

net.ipv4.tcp_timestamps = 0

#内核放弃建立连接之前发送SYNACK 包的数量

net.ipv4.tcp_synack_retries = 1

#内核放弃建立连接之前发送SYN 包的数量

net.ipv4.tcp_syn_retries = 1

#启用timewait 快速回收

net.ipv4.tcp_tw_recycle = 1

#开启重用。允许将TIME-WAIT sockets 重新用于新的TCP 连接

net.ipv4.tcp_tw_reuse = 1

net.ipv4.tcp_mem = 94500000 915000000 927000000

net.ipv4.tcp_fin_timeout = 1

#当keepalive 起用的时候，TCP 发送keepalive 消息的频度。缺省是2 小时

net.ipv4.tcp_keepalive_time = 1800

net.ipv4.tcp_keepalive_probes = 3

net.ipv4.tcp_keepalive_intvl = 15

#允许系统打开的端口范围

net.ipv4.ip_local_port_range = 1024 65000

#修改防火墙表大小，默认65536

net.netfilter.nf_conntrack_max=655350

net.netfilter.nf_conntrack_tcp_timeout_established=1200

# 确保无人能修改路由表

net.ipv4.conf.all.accept_redirects = 0

net.ipv4.conf.default.accept_redirects = 0

net.ipv4.conf.all.secure_redirects = 0

net.ipv4.conf.default.secure_redirects = 0

EOF

#生效

sysctl -p

#用户进程限制

sed -i 's#4096#65535#g' /etc/security/limits.d/20-nproc.conf #加大普通用户限制也可以改为unlimited

#set SSH

sed -i 's/#UseDNS yes/UseDNS no/g' /etc/ssh/sshd_config

sed -i 's/#Port 22/Port 65500/g' /etc/ssh/sshd_config

service sshd restart

#set system files permission

chmod 600 /etc/passwd

chmod 600 /etc/group

chmod 600 /etc/shadow

chmod 600 /etc/gshadow

#set ntp

yum install ntpdate -y

ntpdate ntp.fudan.edu.cn

echo "* 3 * * * /usr/sbin/ntpdate ntp.fudan.edu.cn >/dev/null 2>&1" >>/etc/crontab

service crond restart

check_ok

action "ntpdate is installed and add in crontab" /bin/true

#set vim

echo "===install vim,it will take serval mintinues==="

yum install vim-enhanced -y &>/dev/null

alias vi=vim

echo "alias vi=vim" >>/root/.bashrc

check_ok

action "vim is installed" /bin/true

#set yum repos

echo "===update yum repos,it will take serval mintinues==="

mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

yum clean all &>/dev/null

yum makecache &>/dev/null

check_ok

action "yum repos update is ok" /bin/true

By sean, 8 years ago

未分类

Mac系统下安装ipython分别支持python2和python3

操作系统：Mac10.11.5 python2.7.13 python3.6.1 安装python2：

brew install python

1	brew install python

安装python3：

brew install python3

1	brew install python3

此时，命令行输入python3则打开python3，输入python则打开python2。利用pip安装 ipython

sudo pip install ipython
sudo pip3 install ipython

1 2	sudo pip install ipython sudo pip3 install ipython

用pip安装ipython则是安装到python2的site-packages中，同理用pip3则是安装到python3的site-packages中。进入/usr/local/bin目录下，可以看到ipython文件，用vi打开：

sudo vi ipython

1	sudo vi ipython

#!/usr/local/opt/python/bin/python2.7

# -*- coding: utf-8 -*-
import re
import sys

from IPython import start_ipython

if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
    sys.exit(start_ipython())

#!/usr/local/opt/python/bin/python2.7

# -*- coding: utf-8 -*-

import re

import sys

from IPython import start_ipython

if __name__ == '__main__':

sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])

sys.exit(start_ipython())

#!/usr/local/opt/python/bin/python2.7这个目录就是指向python的版本，
例如我要定向成python3可以修改该成#!/usr/local/opt/python3/bin/python3.6
这样ipython这个命令打开的ipython指向的python版本就是python3了。为了方便我们可以复制这个文件为ipython3，把ipython3这个命令文件指向python3版本，这样在终端输入ipython打开的就是python2，输入ipython3打开的就是python3，是不是很方便！

#!/usr/local/opt/python/bin/python2.7这个目录就是指向python的版本，

例如我要定向成python3可以修改该成#!/usr/local/opt/python3/bin/python3.6

这样ipython这个命令打开的ipython指向的python版本就是python3了。为了方便我们可以复制这个文件为ipython3，把ipython3这个命令文件指向python3版本，这样在终端输入ipython打开的就是python2，输入ipython3打开的就是python3，是不是很方便！

By sean, 8 years10/01/2017 ago

未分类

openvpn服务器防火墙设置

# sample configuration for iptables service
# # you can edit this manually or use system-config-firewall
# # please do not ask us to add additional ports/services to this default configuration
#################
#     NAT       #
#################
*nat
:PREROUTING ACCEPT [27:11935]
:INPUT ACCEPT [0:0]
:OUTPUT ACCEPT [598:57368]
:POSTROUTING ACCEPT [591:57092]
:DOCKER - [0:0]

#所有内网服务器通过本机上网
#-A POSTROUTING -s 10.x.x.0/24 -p tcp -j SNAT --to-source 0.x.x.x

#OpenVPN
-A POSTROUTING -o eth0 -s 10.8.0.0/24 -j MASQUERADE

#-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
#-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
#-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
COMMIT

#################
#     filter    #
#################
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
#:DOCKER - [0:0]

# 这里对已经建立连接的包直接放行，以提高iptables 效率(此条规则通常放在第一条)
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
-A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
#允许ping
-A INPUT -p icmp -m state --state ESTABLISHED,RELATED -j ACCEPT

#本机设备放行
-A INPUT -i lo -j ACCEPT
-A OUTPUT -o lo -j ACCEPT
-A INPUT -s 127.0.0.1 -d 127.0.0.1 -j ACCEPT
-A OUTPUT -s 127.0.0.1 -d 127.0.0.1 -j ACCEPT


#开放sshd服务
-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
-A OUTPUT -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

#为内网机器上网转发
#-A FORWARD -i eth0 -d 10.x.x.0/24 -j ACCEPT
#docker
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 139291:461018923 -j ACCEPT
#-A FORWARD -o docker0 -j DOCKER
#-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
#-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
#-A FORWARD -i docker0 -o docker0 -j ACCEPT

#openvpn
-A INPUT -i eth0 -m state --state NEW -p tcp --dport 1194 -j ACCEPT
-A OUTPUT -p tcp --sport 1194 -m state --state ESTABLISHED -j ACCEPT
#openvpn web ui
-A INPUT -i eth0 -m state --state NEW -p tcp --dport 8080 -j ACCEPT
-A OUTPUT -p tcp --sport 8080 -m state --state ESTABLISHED -j ACCEPT
-A INPUT -i tun+ -j ACCEPT
-A FORWARD -i tun+ -j ACCEPT
-A FORWARD -i tun+ -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i eth0 -o tun+ -m state --state RELATED,ESTABLISHED -j ACCEPT
-A OUTPUT -o tun+ -j ACCEPT

#SMTP
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 25 -j ACCEPT
#允许 80 443 端口，http 和 https
-A INPUT -i eth0 -m state --state NEW -p tcp --dport 80 -j ACCEPT
-A INPUT -i eth0 -m state --state NEW -p tcp --sport 80 -j ACCEPT
-A OUTPUT -p tcp --sport 80 -j ACCEPT
-A OUTPUT -p tcp --dport 80 -j ACCEPT
-A INPUT -i eth0 -m state --state NEW -p tcp --dport 443 -j ACCEPT
-A INPUT -i eth0 -m state --state NEW -p tcp --sport 443 -j ACCEPT
-A OUTPUT -p tcp --sport 443 -j ACCEPT
-A OUTPUT -p tcp --dport 443 -j ACCEPT
#dns
-A OUTPUT -p udp --dport 53 -j ACCEPT
-A INPUT -p udp --sport 53 -j ACCEPT
-A INPUT -p udp --dport 53 -j ACCEPT
-A OUTPUT -p udp --sport 53 -j ACCEPT
#---ganglia的端口----
#ganglia展示web界面、解释php用的apache
-A INPUT -m state --state NEW -m tcp -p tcp --dport 81 -j ACCEPT
-A OUTPUT -p tcp --sport 81 -m state --state ESTABLISHED -j ACCEPT
#ganglia gmond的端口
-A INPUT -p tcp -m multiport --dport 8649:8652 -j ACCEPT
-A OUTPUT -p tcp -m multiport --sport 8649:8652 -m state --state ESTABLISHED -j ACCEPT
-A INPUT -p udp -m multiport --dport 8649:8652 -j ACCEPT
-A OUTPUT -p udp -m multiport --sport 8649:8652 -m state --state ESTABLISHED -j ACCEPT
#falcon-agent sent
-A OUTPUT -p tcp --dport 6030 -j ACCEPT
-A OUTPUT -p tcp --dport 8433 -j ACCEPT
-A INPUT -s 10.x.x.x -p tcp --dport 1988 -j ACCEPT

#nagios的端口
#-A INPUT -p udp -m multiport --dport 5666,12489 -j ACCEPT
#-A OUTPUT -p tcp -m multiport --sport 5666:12489 -m state --state ESTABLISHED -j ACCEPT

###TCP ports for FreeIPA
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 443  -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 389 -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 636 -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 88  -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 464  -j ACCEPT
#-A INPUT -m state --state NEW -m tcp -p tcp --dport 53 -j ACCEPT
###UDP ports for FreeIPA
#-A INPUT -m state --state NEW -m udp -p udp --dport 88 -j ACCEPT
#-A INPUT -m state --state NEW -m udp -p udp --dport 464 -j ACCEPT
#-A INPUT -m state --state NEW -m udp -p udp --dport 123 -j ACCEPT
#-A INPUT -m state --state NEW -m udp -p udp --dport 53 -j ACCEPT
#
##nexus yum
#-A INPUT -m state --state NEW -m tcp -p tcp  --dport 8081 -j ACCEPT

#pptp
#-A INPUT -m state --state NEW -m tcp -p tcp  --dport 1723 -j ACCEPT


#处理IP碎片数量,防止攻击,允许每秒100个
-A FORWARD -f -m limit --limit 100/s --limit-burst 100 -j ACCEPT
#允许来自外部的ping测试
#-A INPUT -p icmp --icmp-type echo-request -j ACCEPT
#禁止ping
-A INPUT -p icmp -i eth0 -j DROP
#设置ICMP包过滤,允许每秒1个包,限制触发条件是10个包
-A FORWARD -p icmp -m limit --limit 1/s --limit-burst 10 -j ACCEPT

##允许登录者
#sean's home
#-A INPUT -s x.x.x.x -m state --state NEW -m tcp -p tcp -j ACCEPT

####################
#DROP RULL
####################
#丢弃坏的TCP包
#-A FORWARD -p TCP ! --syn -m state --state NEW -j DROP

# Reject spoofed packets
-A INPUT -s 169.254.0.0/16 -j DROP
-A INPUT -s 172.16.0.0/12 -j DROP
-A INPUT -s 127.0.0.0/8 -j DROP

-A INPUT -s 224.0.0.0/4 -j DROP
-A INPUT -d 224.0.0.0/4 -j DROP
-A INPUT -s 240.0.0.0/5 -j DROP
-A INPUT -d 240.0.0.0/5 -j DROP
-A INPUT -s 0.0.0.0/8 -j DROP
-A INPUT -d 0.0.0.0/8 -j DROP
-A INPUT -d 239.255.255.0/24 -j DROP
-A INPUT -d 255.255.255.255 -j DROP

# Stop smurf attacks
-A INPUT -p icmp -m icmp --icmp-type address-mask-request -j DROP
-A INPUT -p icmp -m icmp --icmp-type timestamp-request -j DROP
-A INPUT -p icmp -m icmp --icmp-type 8 -j DROP

# Drop all invalid packets
-A INPUT -m state --state INVALID -j DROP
#-A FORWARD -m state --state INVALID -j DROP
-A OUTPUT -m state --state INVALID -j DROP
-A INPUT -m conntrack --ctstate INVALID -j DROP
#drop 3128端口是 squid 的默认端口
-A INPUT -p tcp --dport 3128 -j DROP
# Drop excessive RST packets to avoid smurf attacks
-A INPUT -p tcp -m tcp --tcp-flags RST RST -m limit --limit 2/second --limit-burst 2 -j ACCEPT

# Attempt to block portscans
# Anyone who tried to portscan us is locked out for an entire day.
-A INPUT   -m recent --name portscan --rcheck --seconds 86400 -j DROP
-A FORWARD -m recent --name portscan --rcheck --seconds 86400 -j DROP

# Once the day has passed, remove them from the portscan list
-A INPUT   -m recent --name portscan --remove
-A FORWARD -m recent --name portscan --remove

# These rules add scanners to the portscan list, and log the attempt.
-A INPUT   -p tcp -m tcp --dport 139 -m recent --name portscan --set -j LOG --log-prefix "Portscan:"
-A INPUT   -p tcp -m tcp --dport 139 -m recent --name portscan --set -j DROP

-A FORWARD -p tcp -m tcp --dport 139 -m recent --name portscan --set -j LOG --log-prefix "Portscan:"
-A FORWARD -p tcp -m tcp --dport 139 -m recent --name portscan --set -j DROP


#禁止其他未允许的规则访问
#-A INPUT -j REJECT --reject-with icmp-host-prohibited
#-A FORWARD -j REJECT --reject-with icmp-host-prohibited
-A INPUT -j DROP
-A OUTPUT -j DROP
-A FORWARD -j DROP

COMMIT

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

# sample configuration for iptables service

# # you can edit this manually or use system-config-firewall

# # please do not ask us to add additional ports/services to this default configuration

#################

# NAT #

#################

*nat

:PREROUTING ACCEPT [27:11935]

:INPUT ACCEPT [0:0]

:OUTPUT ACCEPT [598:57368]

:POSTROUTING ACCEPT [591:57092]

:DOCKER - [0:0]

#所有内网服务器通过本机上网

#-A POSTROUTING -s 10.x.x.0/24 -p tcp -j SNAT --to-source 0.x.x.x

#OpenVPN

-A POSTROUTING -o eth0 -s 10.8.0.0/24 -j MASQUERADE

#-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER

#-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER

#-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE

COMMIT

#################

# filter #

#################

*filter

:INPUT ACCEPT [0:0]

:FORWARD ACCEPT [0:0]

:OUTPUT ACCEPT [0:0]

#:DOCKER - [0:0]

# 这里对已经建立连接的包直接放行，以提高iptables 效率(此条规则通常放在第一条)

-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT

-A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

#允许ping

-A INPUT -p icmp -m state --state ESTABLISHED,RELATED -j ACCEPT

#本机设备放行

-A INPUT -i lo -j ACCEPT

-A OUTPUT -o lo -j ACCEPT

-A INPUT -s 127.0.0.1 -d 127.0.0.1 -j ACCEPT

-A OUTPUT -s 127.0.0.1 -d 127.0.0.1 -j ACCEPT

#开放sshd服务

-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT

-A OUTPUT -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

#为内网机器上网转发

#-A FORWARD -i eth0 -d 10.x.x.0/24 -j ACCEPT

#docker

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 139291:461018923 -j ACCEPT

#-A FORWARD -o docker0 -j DOCKER

#-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

#-A FORWARD -i docker0 ! -o docker0 -j ACCEPT

#-A FORWARD -i docker0 -o docker0 -j ACCEPT

#openvpn

-A INPUT -i eth0 -m state --state NEW -p tcp --dport 1194 -j ACCEPT

-A OUTPUT -p tcp --sport 1194 -m state --state ESTABLISHED -j ACCEPT

#openvpn web ui

-A INPUT -i eth0 -m state --state NEW -p tcp --dport 8080 -j ACCEPT

-A OUTPUT -p tcp --sport 8080 -m state --state ESTABLISHED -j ACCEPT

-A INPUT -i tun+ -j ACCEPT

-A FORWARD -i tun+ -j ACCEPT

-A FORWARD -i tun+ -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT

-A FORWARD -i eth0 -o tun+ -m state --state RELATED,ESTABLISHED -j ACCEPT

-A OUTPUT -o tun+ -j ACCEPT

#SMTP

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 25 -j ACCEPT

#允许 80 443 端口，http 和 https

-A INPUT -i eth0 -m state --state NEW -p tcp --dport 80 -j ACCEPT

-A INPUT -i eth0 -m state --state NEW -p tcp --sport 80 -j ACCEPT

-A OUTPUT -p tcp --sport 80 -j ACCEPT

-A OUTPUT -p tcp --dport 80 -j ACCEPT

-A INPUT -i eth0 -m state --state NEW -p tcp --dport 443 -j ACCEPT

-A INPUT -i eth0 -m state --state NEW -p tcp --sport 443 -j ACCEPT

-A OUTPUT -p tcp --sport 443 -j ACCEPT

-A OUTPUT -p tcp --dport 443 -j ACCEPT

#dns

-A OUTPUT -p udp --dport 53 -j ACCEPT

-A INPUT -p udp --sport 53 -j ACCEPT

-A INPUT -p udp --dport 53 -j ACCEPT

-A OUTPUT -p udp --sport 53 -j ACCEPT

#---ganglia的端口----

#ganglia展示web界面、解释php用的apache

-A INPUT -m state --state NEW -m tcp -p tcp --dport 81 -j ACCEPT

-A OUTPUT -p tcp --sport 81 -m state --state ESTABLISHED -j ACCEPT

#ganglia gmond的端口

-A INPUT -p tcp -m multiport --dport 8649:8652 -j ACCEPT

-A OUTPUT -p tcp -m multiport --sport 8649:8652 -m state --state ESTABLISHED -j ACCEPT

-A INPUT -p udp -m multiport --dport 8649:8652 -j ACCEPT

-A OUTPUT -p udp -m multiport --sport 8649:8652 -m state --state ESTABLISHED -j ACCEPT

#falcon-agent sent

-A OUTPUT -p tcp --dport 6030 -j ACCEPT

-A OUTPUT -p tcp --dport 8433 -j ACCEPT

-A INPUT -s 10.x.x.x -p tcp --dport 1988 -j ACCEPT

#nagios的端口

#-A INPUT -p udp -m multiport --dport 5666,12489 -j ACCEPT

#-A OUTPUT -p tcp -m multiport --sport 5666:12489 -m state --state ESTABLISHED -j ACCEPT

###TCP ports for FreeIPA

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 443 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 389 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 636 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 88 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 464 -j ACCEPT

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 53 -j ACCEPT

###UDP ports for FreeIPA

#-A INPUT -m state --state NEW -m udp -p udp --dport 88 -j ACCEPT

#-A INPUT -m state --state NEW -m udp -p udp --dport 464 -j ACCEPT

#-A INPUT -m state --state NEW -m udp -p udp --dport 123 -j ACCEPT

#-A INPUT -m state --state NEW -m udp -p udp --dport 53 -j ACCEPT

##nexus yum

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 8081 -j ACCEPT

#pptp

#-A INPUT -m state --state NEW -m tcp -p tcp --dport 1723 -j ACCEPT

#处理IP碎片数量,防止攻击,允许每秒100个

-A FORWARD -f -m limit --limit 100/s --limit-burst 100 -j ACCEPT

#允许来自外部的ping测试

#-A INPUT -p icmp --icmp-type echo-request -j ACCEPT

#禁止ping

-A INPUT -p icmp -i eth0 -j DROP

#设置ICMP包过滤,允许每秒1个包,限制触发条件是10个包

-A FORWARD -p icmp -m limit --limit 1/s --limit-burst 10 -j ACCEPT

##允许登录者

#sean's home

#-A INPUT -s x.x.x.x -m state --state NEW -m tcp -p tcp -j ACCEPT

####################

#DROP RULL

####################

#丢弃坏的TCP包

#-A FORWARD -p TCP ! --syn -m state --state NEW -j DROP

# Reject spoofed packets

-A INPUT -s 169.254.0.0/16 -j DROP

-A INPUT -s 172.16.0.0/12 -j DROP

-A INPUT -s 127.0.0.0/8 -j DROP

-A INPUT -s 224.0.0.0/4 -j DROP

-A INPUT -d 224.0.0.0/4 -j DROP

-A INPUT -s 240.0.0.0/5 -j DROP

-A INPUT -d 240.0.0.0/5 -j DROP

-A INPUT -s 0.0.0.0/8 -j DROP

-A INPUT -d 0.0.0.0/8 -j DROP

-A INPUT -d 239.255.255.0/24 -j DROP

-A INPUT -d 255.255.255.255 -j DROP

# Stop smurf attacks

-A INPUT -p icmp -m icmp --icmp-type address-mask-request -j DROP

-A INPUT -p icmp -m icmp --icmp-type timestamp-request -j DROP

-A INPUT -p icmp -m icmp --icmp-type 8 -j DROP

# Drop all invalid packets

-A INPUT -m state --state INVALID -j DROP

#-A FORWARD -m state --state INVALID -j DROP

-A OUTPUT -m state --state INVALID -j DROP

-A INPUT -m conntrack --ctstate INVALID -j DROP

#drop 3128端口是 squid 的默认端口

-A INPUT -p tcp --dport 3128 -j DROP

# Drop excessive RST packets to avoid smurf attacks

-A INPUT -p tcp -m tcp --tcp-flags RST RST -m limit --limit 2/second --limit-burst 2 -j ACCEPT

# Attempt to block portscans

# Anyone who tried to portscan us is locked out for an entire day.

-A INPUT -m recent --name portscan --rcheck --seconds 86400 -j DROP

-A FORWARD -m recent --name portscan --rcheck --seconds 86400 -j DROP

# Once the day has passed, remove them from the portscan list

-A INPUT -m recent --name portscan --remove

-A FORWARD -m recent --name portscan --remove

# These rules add scanners to the portscan list, and log the attempt.

-A INPUT -p tcp -m tcp --dport 139 -m recent --name portscan --set -j LOG --log-prefix "Portscan:"

-A INPUT -p tcp -m tcp --dport 139 -m recent --name portscan --set -j DROP

-A FORWARD -p tcp -m tcp --dport 139 -m recent --name portscan --set -j LOG --log-prefix "Portscan:"

-A FORWARD -p tcp -m tcp --dport 139 -m recent --name portscan --set -j DROP

#禁止其他未允许的规则访问

#-A INPUT -j REJECT --reject-with icmp-host-prohibited

#-A FORWARD -j REJECT --reject-with icmp-host-prohibited

-A INPUT -j DROP

-A OUTPUT -j DROP

-A FORWARD -j DROP

COMMIT

By sean, 8 years09/28/2017 ago

未分类

centos用 yum 方式安装 nodejs 和 npm

centos用 yum 方式安装 nodejs 和 npm 要通过 yum 来安装 nodejs 和 npm 需要先给 yum 添加 epel 源，添加方法在 centos 添加epel和remi源中安装完成后,执行

yum -y install nodejs npm --enablerepo=epel

1	yum -y install nodejs npm --enablerepo=epel

注：centos 添加 epel 和 remi 源添加 epel 源 64位:

rpm -ivh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm

1 2	rpm -ivh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm

32位:

rpm -ivh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm

1 2	rpm -ivh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm

导入 key:

rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6

1 2	rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6

添加 remi 源

 rpm -ivh http://rpms.famillecollet.com/enterprise/remi-release-6.rpm
 rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-remi

1 2	rpm -ivh http://rpms.famillecollet.com/enterprise/remi-release-6.rpm rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-remi

问题解决： yum 安装完node版本是6.17.1，通过npm install –registry=https://registry.npm.taobao.org 安装时会报以下错误： fetchMetadata: Read more…

By sean, 8 years09/21/2017 ago

未分类

安装ipa-client错误：kinit: Clients credentials have been revoked while getting initial credentials

安装ipa-client时，遇到如下错误：

[root@log1-prod-iuap-yycloud scripts]# ipa-client-install --mkhomedir --enable-dns-updates --force-join --principal=admin --password=xxxxxx -U
Discovery was successful!
Client hostname: log1-prod-iuap-yycloud.yonyouiuap.com
Realm: YONYOUIUAP.COM
DNS Domain: yonyouiuap.com
IPA Server: freeipa4-iuap-hb2-ali.yonyouiuap.com
BaseDN: dc=yonyouiuap,dc=com

Synchronizing time with KDC...
Attempting to sync time using ntpd.  Will timeout after 15 seconds
Please make sure the following ports are opened in the firewall settings:
     TCP: 80, 88, 389
     UDP: 88 (at least one of TCP/UDP ports 88 has to be open)
Also note that following ports are necessary for ipa-client working properly after enrollment:
     TCP: 464
     UDP: 464, 123 (if NTP enabled)
Kerberos authentication failed: kinit: Clients credentials have been revoked while getting initial credentials

Installation failed. Rolling back changes.
IPA client is not configured on this system.

[root@log1-prod-iuap-yycloud scripts]# ipa-client-install --mkhomedir --enable-dns-updates --force-join --principal=admin --password=xxxxxx -U

Discovery was successful!

Client hostname: log1-prod-iuap-yycloud.yonyouiuap.com

Realm: YONYOUIUAP.COM

DNS Domain: yonyouiuap.com

IPA Server: freeipa4-iuap-hb2-ali.yonyouiuap.com

BaseDN: dc=yonyouiuap,dc=com

Synchronizing time with KDC...

Attempting to sync time using ntpd. Will timeout after 15 seconds

Please make sure the following ports are opened in the firewall settings:

TCP: 80, 88, 389

UDP: 88 (at least one of TCP/UDP ports 88 has to be open)

Also note that following ports are necessary for ipa-client working properly after enrollment:

TCP: 464

UDP: 464, 123 (if NTP enabled)

Kerberos authentication failed: kinit: Clients credentials have been revoked while getting initial credentials

Installation failed. Rolling back changes.

IPA client is not configured on this system.

执行kinit admin:

[sean@middlewaredocker1-iuap-hb2-ali ~]$ kinit admin
kinit: Clients credentials have been revoked while getting initial credentials

1 2	[sean@middlewaredocker1-iuap-hb2-ali ~]$ kinit admin kinit: Clients credentials have been revoked while getting initial credentials

去ipa server端查询：

[root@freeipa4-iuap-hb2-ali sean]# ipa user-status admin
-----------------------
Account disabled: False
-----------------------
  Server: freeipa3-iuap-hb2-ali.yonyouiuap.com
  Failed logins: 6
  Last successful authentication: 20170918090442Z
  Last failed authentication: 20170918115216Z
  Time now: 2017-09-18T12:04:36Z

  Server: freeipa1-iuap-hb2-ali.yonyouiuap.com failed: Server is unwilling to perform: Too many failed logins.

  Server: freeipa4-iuap-hb2-ali.yonyouiuap.com
  Failed logins: 0
  Last successful authentication: 20170918120436Z
  Last failed authentication: 20170918115237Z
  Time now: 2017-09-18T12:04:36Z
----------------------------
Number of entries returned 3
----------------------------

[root@freeipa4-iuap-hb2-ali sean]# ipa user-status admin

-----------------------

Account disabled: False

-----------------------

Server: freeipa3-iuap-hb2-ali.yonyouiuap.com

Failed logins: 6

Last successful authentication: 20170918090442Z

Last failed authentication: 20170918115216Z

Time now: 2017-09-18T12:04:36Z

Server: freeipa1-iuap-hb2-ali.yonyouiuap.com failed: Server is unwilling to perform: Too many failed logins.

Server: freeipa4-iuap-hb2-ali.yonyouiuap.com

Failed logins: 0

Last successful authentication: 20170918120436Z

Last failed authentication: 20170918115237Z

Time now: 2017-09-18T12:04:36Z

----------------------------

Number of entries returned 3

----------------------------

原来是尝试次数过多（默认6次）被锁了解锁一下：

----------------------------
[root@freeipa4-iuap-hb2-ali sean]# ipa user-unlock admin
------------------------
Unlocked account "admin"
------------------------
[root@freeipa4-iuap-hb2-ali sean]# ipa user-status admin
-----------------------
Account disabled: False
-----------------------
  Server: freeipa3-iuap-hb2-ali.yonyouiuap.com
  Failed logins: 0
  Last successful authentication: 20170922031611Z
  Last failed authentication: 20170918123414Z
  Time now: 2017-09-22T03:16:12Z

  Server: freeipa1-iuap-hb2-ali.yonyouiuap.com
  Failed logins: 0
  Last successful authentication: 20170922031612Z
  Last failed authentication: 20170918123627Z
  Time now: 2017-09-22T03:16:12Z

  Server: freeipa4-iuap-hb2-ali.yonyouiuap.com
  Failed logins: 0
  Last successful authentication: 20170922031611Z
  Last failed authentication: 20170918123621Z
  Time now: 2017-09-22T03:16:12Z
----------------------------
Number of entries returned 3
----------------------------

----------------------------

[root@freeipa4-iuap-hb2-ali sean]# ipa user-unlock admin

------------------------

Unlocked account "admin"

------------------------

[root@freeipa4-iuap-hb2-ali sean]# ipa user-status admin

-----------------------

Account disabled: False

-----------------------

Server: freeipa3-iuap-hb2-ali.yonyouiuap.com

Failed logins: 0

Last successful authentication: 20170922031611Z

Last failed authentication: 20170918123414Z

Time now: 2017-09-22T03:16:12Z

Server: freeipa1-iuap-hb2-ali.yonyouiuap.com

Failed logins: 0

Last successful authentication: 20170922031612Z

Last failed authentication: 20170918123627Z

Time now: 2017-09-22T03:16:12Z

Server: freeipa4-iuap-hb2-ali.yonyouiuap.com

Failed logins: 0

Last successful authentication: 20170922031611Z

Last failed authentication: 20170918123621Z

Time now: 2017-09-22T03:16:12Z

----------------------------

Number of entries returned 3

----------------------------

再次安装ipa-client，成功。

By sean, 8 years09/18/2017 ago