How to Specify an Authenticated Proxy for a Python Http Connection

How to specify an authenticated proxy for a python http connection?

Use this:

import requests

proxies = {"http":"http://username:password@proxy_ip:proxy_port"}

r = requests.get("http://www.example.com/", proxies=proxies)

print(r.content)

I think it's much simpler than using urllib. I don't understand why people love using urllib so much.

How to apply proxy with authentication using http.client for API

I ended up using the requests package, they make it very simple to channel your request through a proxy, sample code below for reference:

import requests

proxies = {
    'http':  f"http://username:pwd@webproxy.subdomain.website.com:8080",
    'https': f"https://username:pwd@webproxy.subdomain.website.com:8080"
}
url = 'https://api.cognitive.microsoft.com/bing/v7.0/entities/'
# query string parameters

params = 'mkt=' + 'en-US' + '&q=' + urllib.parse.quote (query)

# custom headers
headers = {'Ocp-Apim-Subscription-Key': subscriptionKey}

#start session
session = requests.Session()
#persist proxy across request
session.proxies = proxies

# make GET request
r = session.get(url, params=params, headers=headers)

How to pass proxy-authentication (requires digest auth) by using python requests module

I wrote the class that can be used in proxy authentication (based on digest auth).

I borrowed almost all codes from requests.auth.HTTPDigestAuth.

import requests
import requests.auth

class HTTPProxyDigestAuth(requests.auth.HTTPDigestAuth):
    def handle_407(self, r):
        """Takes the given response and tries digest-auth, if needed."""

        num_407_calls = r.request.hooks['response'].count(self.handle_407)

        s_auth = r.headers.get('Proxy-authenticate', '')

        if 'digest' in s_auth.lower() and num_407_calls < 2:

            self.chal = requests.auth.parse_dict_header(s_auth.replace('Digest ', ''))

            # Consume content and release the original connection
            # to allow our new request to reuse the same one.
            r.content
            r.raw.release_conn()

            r.request.headers['Authorization'] = self.build_digest_header(r.request.method, r.request.url)
            r.request.send(anyway=True)
            _r = r.request.response
            _r.history.append(r)

            return _r

        return r

    def __call__(self, r):
        if self.last_nonce:
            r.headers['Proxy-Authorization'] = self.build_digest_header(r.method, r.url)
        r.register_hook('response', self.handle_407)
        return r

Usage:

proxies = {
    "http" :"192.168.20.130:8080",
    "https":"192.168.20.130:8080",
}
auth = HTTPProxyDigestAuth("username", "password")

# HTTP
r = requests.get("http://www.google.co.jp/", proxies=proxies, auth=auth)
r.status_code # 200 OK

# HTTPS
r = requests.get("https://www.google.co.jp/", proxies=proxies, auth=auth)
r.status_code # 200 OK

How to pass proxy-authentication to python Requests module

I found the solution, about how to set proxy authentication globally:

import requests
import urllib2
import re
from requests.auth import HTTPProxyAuth

s = requests.Session()

proxies = {
  "http": "http://185.165.193.208:8000",
  "https": "https://185.165.193.208:8000"
}

auth = HTTPProxyAuth("gRRT7n", "NMu8g0")

s.proxies = proxies
s.auth = auth        # Set authorization parameters globally

ext_ip = s.get('http://checkip.dyndns.org')
print ext_ip.text

BR,
Federico

Python request with authenticated proxy without exposing username and password.

From the request docs:

You can also configure proxies by setting the environment variables
HTTP_PROXY and HTTPS_PROXY.

So you could avoid revealing the credentials in the script by setting the environment variable instead:

$ export HTTPS_PROXY="http://<UserName>:<Password>@proxy_url:8080"

And then in the script you would use the proxy by just calling:

resp = s.get(url, params={'key':urlkey }) 

how to set proxy with authentication in selenium chromedriver python?

Selenium Chrome Proxy Authentication

Setting chromedriver proxy with Selenium using Python

If you need to use a proxy with python and Selenium library with chromedriver you usually use the following code (Without any username and password:

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % hostname + ":" + port)
driver = webdriver.Chrome(chrome_options=chrome_options)

It works fine unless proxy requires authentication. if the proxy requires you to log in with a username and password it will not work. In this case, you have to use more tricky solution that is explained below. By the way, if you whitelist your server IP address from the proxy provider or server it should not ask proxy credentials.

HTTP Proxy Authentication with Chromedriver in Selenium

To set up proxy authentication we will generate a special file and upload it to chromedriver dynamically using the following code below. This code configures selenium with chromedriver to use HTTP proxy that requires authentication with user/password pair.

import os
import zipfile

from selenium import webdriver

PROXY_HOST = '192.168.3.2'  # rotating proxy or host
PROXY_PORT = 8080 # port
PROXY_USER = 'proxy-user' # username
PROXY_PASS = 'proxy-password' # password

manifest_json = """
{
    "version": "1.0.0",
    "manifest_version": 2,
    "name": "Chrome Proxy",
    "permissions": [
        "proxy",
        "tabs",
        "unlimitedStorage",
        "storage",
        "<all_urls>",
        "webRequest",
        "webRequestBlocking"
    ],
    "background": {
        "scripts": ["background.js"]
    },
    "minimum_chrome_version":"22.0.0"
}
"""

background_js = """
var config = {
        mode: "fixed_servers",
        rules: {
        singleProxy: {
            scheme: "http",
            host: "%s",
            port: parseInt(%s)
        },
        bypassList: ["localhost"]
        }
    };

chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

function callbackFn(details) {
    return {
        authCredentials: {
            username: "%s",
            password: "%s"
        }
    };
}

chrome.webRequest.onAuthRequired.addListener(
            callbackFn,
            {urls: ["<all_urls>"]},
            ['blocking']
);
""" % (PROXY_HOST, PROXY_PORT, PROXY_USER, PROXY_PASS)

def get_chromedriver(use_proxy=False, user_agent=None):
    path = os.path.dirname(os.path.abspath(__file__))
    chrome_options = webdriver.ChromeOptions()
    if use_proxy:
        pluginfile = 'proxy_auth_plugin.zip'

        with zipfile.ZipFile(pluginfile, 'w') as zp:
            zp.writestr("manifest.json", manifest_json)
            zp.writestr("background.js", background_js)
        chrome_options.add_extension(pluginfile)
    if user_agent:
        chrome_options.add_argument('--user-agent=%s' % user_agent)
    driver = webdriver.Chrome(
        os.path.join(path, 'chromedriver'),
        chrome_options=chrome_options)
    return driver

def main():
    driver = get_chromedriver(use_proxy=True)
    #driver.get('https://www.google.com/search?q=my+ip+address')
    driver.get('https://httpbin.org/ip')

if __name__ == '__main__':
    main()

Function get_chromedriver returns configured selenium webdriver that you can use in your application. This code is tested and works just fine.

Read more about onAuthRequired event in Chrome.

Establishing a TCP connection through an authenticated HTTP proxy?

As I didn't find any actual modules or other code that I could use, I ended up writing my own function that connects through the proxy:

def http_proxy_connect(address, proxy = None, auth = None, headers = {}):
  """
  Establish a socket connection through an HTTP proxy.

  Arguments:
    address (required)     = The address of the target
    proxy (def: None)      = The address of the proxy server
    auth (def: None)       = A tuple of the username and password used for authentication
    headers (def: {})      = A set of headers that will be sent to the proxy

  Returns:
    A 3-tuple of the format:
      (socket, status_code, headers)
    Where `socket' is the socket object, `status_code` is the HTTP status code that the server
     returned and `headers` is a dict of headers that the server returned.
  """
  import socket
  import base64

  def valid_address(addr):
    """ Verify that an IP/port tuple is valid """
    return isinstance(addr, (list, tuple)) and len(addr) == 2 and isinstance(addr[0], str) and isinstance(addr[1], (int, long))

  if not valid_address(address):
    raise ValueError('Invalid target address')

  if proxy == None:
    s = socket.socket()
    s.connect(address)
    return s, 0, {}

  if not valid_address(proxy):
    raise ValueError('Invalid proxy address')

  headers = {
    'host': address[0]
  }

  if auth != None:
    if isinstance(auth, str):
      headers['proxy-authorization'] = auth
    elif auth and isinstance(auth, (tuple, list)):
      if len(auth) == 1:
        raise ValueError('Invalid authentication specification')

      t = auth[0]
      args = auth[1:]

      if t.lower() == 'basic' and len(args) == 2:
        headers['proxy-authorization'] = 'Basic ' + base64.b64encode('%s:%s' % args)
      else:
        raise ValueError('Invalid authentication specification')
    else:
      raise ValueError('Invalid authentication specification')

  s = socket.socket()
  s.connect(proxy)
  fp = s.makefile('r+')

  fp.write('CONNECT %s:%d HTTP/1.0\r\n' % address)
  fp.write('\r\n'.join('%s: %s' % (k, v) for (k, v) in headers.items()) + '\r\n\r\n')
  fp.flush()

  statusline = fp.readline().rstrip('\r\n')

  if statusline.count(' ') < 2:
    fp.close()
    s.close()
    raise IOError('Bad response')
  version, status, statusmsg = statusline.split(' ', 2)
  if not version in ('HTTP/1.0', 'HTTP/1.1'):
    fp.close()
    s.close()
    raise IOError('Unsupported HTTP version')
  try:
    status = int(status)
  except ValueError:
    fp.close()
    s.close()
    raise IOError('Bad response')

  response_headers = {}

  while True:
    tl = ''
    l = fp.readline().rstrip('\r\n')
    if l == '':
      break
    if not ':' in l:
      continue
    k, v = l.split(':', 1)
    response_headers[k.strip().lower()] = v.strip()

  fp.close()
  return (s, status, response_headers)

---

Related Topics