1. 在centos7上安装docker
#安装docker yum install docker -y #安装docker阿里云加速器,下面的PRIVATE_ID请换成自己的docker的仓库id sudo mkdir -p /etc/docker sudo tee /etc/docker/daemon.json <<-'EOF' { "registry-mirrors": ["https://PRIVATE_ID.mirror.aliyuncs.com"] } EOF sudo systemctl daemon-reload #启动docker systemctl start docker #设置系统启动加载docker服务 chkconfig docker on
2. 下载最新centos镜像并运行容器
#下载centos:latest docker pull docker.io/centos:latest #查看image id = 84c2xxxx docker images #创建共享目录 mkdir -p /root/docker/shared #运行容器,预留ssh 22端口映射为10022;预留mariaDB 3306映射为13306 docker run -p 10022:22 -p 13306:3306 --privileged --name centos -v /root/docker/shared:/mnt/shared -it docker.io/centos:latest /sbin/init
3. 在镜像中安装scrapy爬虫环境
#安装必要工具 yum install vim openssl-devel bzip2-devel expat-devel gdbm-devel readline-devel sqlite-devel wget -y yum install -y gcc gcc-c++ autoconf automake libtool make #下载python3cdwget https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz#解压并安装python3tar zxvf Python-3.6.4.tgzcd Python-3.6.4 ./configure --prefix=/usr/local/python3 --enable-optimizations make && make install ln -s /usr/local/python3/bin/python3.6 /usr/bin/python3 ln -s /usr/local/python3/bin/pip /usr/bin/pip3y
4. 备份python2.7并修正yum工具中python指向
#备份python2.7环境 mv /usr/bin/python /usr/bin/python.bak ls -s /usr/bin/python3 /usr/bin/python mv /usr/bin/pip /usr/bin/pip.bak ls -s /usr/bin/pip3y /usr/bin/pip #修改yum中的python环境指向 vim /usr/bin/yum #将第一行 python改为python2.7 vim /usr/libexec/urlgrabber-ext-down #将第一行 python改为python2.7
5. 安装scrapy
#安装scrapy pip install scrapy #部署路径 ln -s /usr/local/python3/bin/scrapy /usr/bin/scrapy
6. 打包image并上传阿里云供后续备用
#在docker中退出container exit #在操作系统中查看container id docker ps -a #查到container id:9023xxx,打包为:scrapy:v1 docker commit 9023 scrapy:v1 #查看image id为8bdc docker images #将image更新/保存到阿里云 sudo docker login --username=ACCOUNT registry.cn-hangzhou.aliyuncs.com sudo docker tag 8dbc registry.cn-hangzhou.aliyuncs.com/norble/scrapy:v1 sudo docker push registry.cn-hangzhou.aliyuncs.com/norble/scrapy:v1