2018 Scrapy Environment Enhance(4)Docker Service for Scrapyd and Tor Network

2018ScrapyEnvironmentEnhance(4)DockerServiceforScrapydandTorNetwork

DockerServiceforScrapyd

DockerServiceforTorNetwork

Thereare3portsand3differentfunctionservices.

Tor9051PortresetIP

Tor9050Porttoproxy

Privoxy8118toproxyto9050

IputthemintooneDockerApp,butweshouldbeabletosplittheminto2apps.

Dockerfile

#RunaTorNetworkServer

#PreparetheOS

FROMubuntu:16.04

MAINTAINERCarlLuo<[email protected]>

ENVDEBIAN_FRONTENDnoninteractive

RUNapt-get-qqupdate

RUNapt-get-qqydist-upgrade

#Preparethedenpendencies

RUNapt-getinstall-qytor

RUNapt-getinstall-qynetcat

RUNapt-getinstall-qycurl

RUNapt-getinstall-qyprivoxy

#setuptorconfiguration

RUNecho"ControlPort9051">>/etc/tor/torrc

RUNecho"ControlListenAddress0.0.0.0">>/etc/tor/torrc

RUNechoHashedControlPassword$(tor--hash-password“xxxxxxxx"|tail-n1)>>/etc/tor/torrc

#setupforwardconfiguration

RUNecho"forward-socks5t/127.0.0.1:9050.">>/etc/privoxy/config

RUNecho"listen-address0.0.0.0:8118">>/etc/privoxy/config

#setuptheapp

EXPOSE90518118

RUNmkdir-p/app/

ADDstart.sh/app/

WORKDIR/app/

CMD["./start.sh"]

Thecommandlinetostart,start.sh

#!/bin/sh-ex

#starttheservice

servicetorstart

serviceprivoxystart

tail-f/dev/null

MakefiletoopenthePORTnumber

IMAGE=sillycat/public

TAG=ubuntu-tornetwork-1.0

NAME=ubuntu-tornetwork-1.0

docker-context:

build:docker-context

dockerbuild-t$(IMAGE):$(TAG).

run:

dockerrun-d-p9051:9051-p8118:8118--name$(NAME)$(IMAGE):$(TAG)

debug:

dockerrun-p9051:9051-p8118:8118--name$(NAME)-ti$(IMAGE):$(TAG)/bin/bash

clean:

dockerstop${NAME}

dockerrm${NAME}

logs:

dockerlogs${NAME}

publish:

dockerpush${IMAGE}

HowtochangetheIP

>echo-e'AUTHENTICATE“xxxxxxxxx"\r\nsignalNEWNYM\r\nQUIT'|nclocalhost9051

ChecktheIP

>curl-xlocalhost:8118http://icanhazip.com/

HowtochangetheIPfromremote

>echo-e'AUTHENTICATE“xxxxxxxxx"\r\nsignalNEWNYM\r\nQUIT'|ncubuntu-master9051

ChecktheIPfromRemote

>curl-xubuntu-master:8118http://icanhazip.com/

References:

https://github.com/torproject/tor/blob/master/src/config/torrc.sample.in

https://github.com/osminogin/docker-tor-simple

https://github.com/dperson/torproxy

https://program-think.blogspot.com/2014/12/gfw-privoxy.html

http://www.cnblogs.com/another-wheel/archive/2011/11/16/setup-http-proxy-via-socks-and-privoxy.html

https://blog.phpgao.com/privoxy-shadowsocks.html

https://blog.csdn.net/north_eagle/article/details/8683403

https://stackoverflow.com/questions/15459170/trouble-authenticating-tor-with-python

https://gist.github.com/DusanMadar/c1155329cf6a71e4346cae271a2eafd3

https://www.torproject.org/docs/faq.html.en

https://stackoverflow.com/questions/45901892/how-to-connect-to-tor-control-port-9051-from-a-remote-host

https://stackoverflow.com/questions/25775266/how-to-keep-docker-container-running-after-starting-services/36872226