[957af0d] | 1 | #!/usr/bin/env python |
---|
| 2 | # -*- coding: utf-8 -*- |
---|
[a1b8fee] | 3 | from __future__ import print_function |
---|
| 4 | |
---|
[957af0d] | 5 | import sys |
---|
| 6 | import json |
---|
| 7 | import logging |
---|
| 8 | import re |
---|
| 9 | |
---|
[5251ec6] | 10 | try: |
---|
| 11 | # CRUFT: python 3 uses urllib.request instead of urllib2 |
---|
| 12 | import urllib2 |
---|
| 13 | except ImportError: |
---|
| 14 | from urllib import request as urllib2 |
---|
[c155a16] | 15 | |
---|
[463e7ffc] | 16 | logger = logging.getLogger(__name__) |
---|
[c155a16] | 17 | |
---|
[957af0d] | 18 | ''' |
---|
| 19 | HTTP Proxy parser and Connection |
---|
| 20 | |
---|
| 21 | connect() function: |
---|
| 22 | - auto detects proxy in windows, osx |
---|
[20fa5fe] | 23 | - in ux systems, the http_proxy environment variable must be set |
---|
[957af0d] | 24 | - if it fails, try to find the proxy.pac address. |
---|
| 25 | - parses the file, and looks up for all possible proxies |
---|
| 26 | ''' |
---|
| 27 | |
---|
| 28 | |
---|
| 29 | class Connection(object): |
---|
| 30 | |
---|
| 31 | def __init__(self, url, timeout): |
---|
| 32 | self.url = url |
---|
| 33 | self.timeout = timeout |
---|
| 34 | |
---|
| 35 | def _get_addresses_of_proxy_pac(self): |
---|
| 36 | """ |
---|
| 37 | Return a list of possible auto proxy .pac files being used, |
---|
| 38 | based on the system registry (win32) or system preferences (OSX). |
---|
| 39 | @return: list of urls |
---|
| 40 | """ |
---|
| 41 | pac_files = [] |
---|
| 42 | if sys.platform == 'win32': |
---|
| 43 | try: |
---|
| 44 | import _winreg as winreg # used from python 2.0-2.6 |
---|
| 45 | except: |
---|
| 46 | import winreg # used from python 2.7 onwards |
---|
| 47 | net = winreg.OpenKey( |
---|
| 48 | winreg.HKEY_CURRENT_USER, |
---|
| 49 | "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings" |
---|
| 50 | ) |
---|
| 51 | n_subs, n_vals, last_mod = winreg.QueryInfoKey(net) |
---|
| 52 | subkeys = {} |
---|
| 53 | for i in range(n_vals): |
---|
| 54 | this_name, this_val, this_type = winreg.EnumValue(net, i) |
---|
| 55 | subkeys[this_name] = this_val |
---|
[5251ec6] | 56 | if 'AutoConfigURL' in subkeys and len(subkeys['AutoConfigURL']) > 0: |
---|
[957af0d] | 57 | pac_files.append(subkeys['AutoConfigURL']) |
---|
| 58 | elif sys.platform == 'darwin': |
---|
| 59 | import plistlib |
---|
| 60 | sys_prefs = plistlib.readPlist( |
---|
| 61 | '/Library/Preferences/SystemConfiguration/preferences.plist') |
---|
| 62 | networks = sys_prefs['NetworkServices'] |
---|
| 63 | # loop through each possible network (e.g. Ethernet, Airport...) |
---|
| 64 | for network in networks.items(): |
---|
| 65 | # the first part is a long identifier |
---|
| 66 | net_key, network = network |
---|
[5251ec6] | 67 | if 'ProxyAutoConfigURLString' in network['Proxies']: |
---|
[957af0d] | 68 | pac_files.append( |
---|
| 69 | network['Proxies']['ProxyAutoConfigURLString']) |
---|
| 70 | return list(set(pac_files)) # remove redundant ones |
---|
| 71 | |
---|
| 72 | def _parse_proxy_pac(self, pac_urls_list): |
---|
| 73 | ''' |
---|
| 74 | For every pac file url in pac_urls_list, it tryes to connect. |
---|
| 75 | If the connection is successful parses the file in search for |
---|
| 76 | http proxies. |
---|
| 77 | @param pac_urls_list: List with urls for the pac files |
---|
| 78 | @return: list with all found http proxies |
---|
| 79 | ''' |
---|
| 80 | proxy_url_list = [] |
---|
| 81 | for this_pac_url in pac_urls_list: |
---|
[c155a16] | 82 | logger.debug('Trying pac file (%s)...' % this_pac_url) |
---|
[957af0d] | 83 | try: |
---|
| 84 | response = urllib2.urlopen( |
---|
| 85 | this_pac_url, timeout=self.timeout) |
---|
[c155a16] | 86 | logger.debug('Succeeded (%s)...' % this_pac_url) |
---|
[957af0d] | 87 | except Exception: |
---|
[c155a16] | 88 | logger.debug('Failled (%s)...' % this_pac_url) |
---|
[957af0d] | 89 | continue |
---|
| 90 | pacStr = response.read() |
---|
| 91 | possProxies = re.findall( |
---|
| 92 | r"PROXY\s([^\s;,:]+:[0-9]{1,5})[^0-9]", pacStr + '\n') |
---|
| 93 | for thisPoss in possProxies: |
---|
| 94 | prox_url = 'http://' + thisPoss |
---|
| 95 | proxy_dic = {'http': prox_url} |
---|
| 96 | proxy_url_list.append(proxy_dic) |
---|
| 97 | return proxy_url_list |
---|
| 98 | |
---|
| 99 | def _set_proxy(self,proxy_dic=None): |
---|
| 100 | ''' |
---|
| 101 | Sets connection proxy. |
---|
| 102 | if proxy_dic is None get's teh proxy from the system. |
---|
| 103 | To disable autodetected proxy pass an empty dictionary: {} |
---|
| 104 | @param proxy_dic: format: {'http': 'http://www.example.com:3128/'} |
---|
| 105 | ''' |
---|
| 106 | if proxy_dic is None: |
---|
| 107 | # The default is to read the list of proxies from the environment variables <protocol>_proxy. |
---|
| 108 | # If no proxy environment variables are set, then in a Windows environment proxy settings are |
---|
| 109 | # obtained from the registry's Internet Settings section, and in a Mac OS X environment proxy |
---|
| 110 | # information is retrieved from the OS X System Configuration |
---|
| 111 | # Framework. |
---|
| 112 | proxy = urllib2.ProxyHandler() |
---|
| 113 | else: |
---|
| 114 | # If proxies is given, it must be a dictionary mapping protocol names to |
---|
| 115 | # URLs of proxies. |
---|
| 116 | proxy = urllib2.ProxyHandler(proxy_dic) |
---|
| 117 | opener = urllib2.build_opener(proxy) |
---|
| 118 | urllib2.install_opener(opener) |
---|
| 119 | |
---|
| 120 | |
---|
| 121 | |
---|
| 122 | |
---|
| 123 | def connect(self): |
---|
| 124 | ''' |
---|
| 125 | Performs the request and gets a response from self.url |
---|
| 126 | @return: response object from urllib2.urlopen |
---|
| 127 | ''' |
---|
| 128 | req = urllib2.Request(self.url) |
---|
| 129 | response = None |
---|
| 130 | try: |
---|
[c155a16] | 131 | logger.debug("Trying Direct connection to %s..."%self.url) |
---|
[957af0d] | 132 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
[5251ec6] | 133 | except Exception as exc: |
---|
[c155a16] | 134 | logger.debug("Failed!") |
---|
[5251ec6] | 135 | logger.debug(exc) |
---|
[957af0d] | 136 | try: |
---|
[c155a16] | 137 | logger.debug("Trying to use system proxy if it exists...") |
---|
[957af0d] | 138 | self._set_proxy() |
---|
| 139 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
[5251ec6] | 140 | except Exception as exc: |
---|
[c155a16] | 141 | logger.debug("Failed!") |
---|
[5251ec6] | 142 | logger.debug(exc) |
---|
[957af0d] | 143 | pac_urls = self._get_addresses_of_proxy_pac() |
---|
| 144 | proxy_urls = self._parse_proxy_pac(pac_urls) |
---|
| 145 | for proxy in proxy_urls: |
---|
| 146 | try: |
---|
[c155a16] | 147 | logger.debug("Trying to use the proxy %s found in proxy.pac configuration"%proxy) |
---|
[957af0d] | 148 | self._set_proxy(proxy) |
---|
| 149 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
[5251ec6] | 150 | except Exception as exc: |
---|
[c155a16] | 151 | logger.debug("Failed!") |
---|
[5251ec6] | 152 | logger.debug(exc) |
---|
[957af0d] | 153 | if response is not None: |
---|
[c155a16] | 154 | logger.debug("The connection to %s was successful."%self.url) |
---|
[957af0d] | 155 | else: |
---|
[c155a16] | 156 | logger.warning("Connection to %s failed..."%self.url) |
---|
[957af0d] | 157 | return response |
---|
| 158 | |
---|
| 159 | |
---|
| 160 | if __name__ == "__main__": |
---|
| 161 | from pprint import pprint |
---|
| 162 | c = Connection() |
---|
| 163 | response = c.connect() |
---|
| 164 | if response is not None: |
---|
[9c3d784] | 165 | print(50 * '-') |
---|
[957af0d] | 166 | content = json.loads(response.read().strip()) |
---|
| 167 | pprint(content) |
---|