1 | #!/usr/bin/env python |
---|
2 | # -*- coding: utf-8 -*- |
---|
3 | import urllib2 |
---|
4 | import sys |
---|
5 | import json |
---|
6 | import logging |
---|
7 | import re |
---|
8 | |
---|
9 | ''' |
---|
10 | HTTP Proxy parser and Connection |
---|
11 | |
---|
12 | connect() function: |
---|
13 | - auto detects proxy in windows, osx |
---|
14 | - in ux systems, the http_proxy enviroment variable must be set |
---|
15 | - if it fails, try to find the proxy.pac address. |
---|
16 | - parses the file, and looks up for all possible proxies |
---|
17 | ''' |
---|
18 | |
---|
19 | |
---|
20 | class Connection(object): |
---|
21 | |
---|
22 | def __init__(self, url, timeout): |
---|
23 | self.url = url |
---|
24 | self.timeout = timeout |
---|
25 | |
---|
26 | def _get_addresses_of_proxy_pac(self): |
---|
27 | """ |
---|
28 | Return a list of possible auto proxy .pac files being used, |
---|
29 | based on the system registry (win32) or system preferences (OSX). |
---|
30 | @return: list of urls |
---|
31 | """ |
---|
32 | pac_files = [] |
---|
33 | if sys.platform == 'win32': |
---|
34 | try: |
---|
35 | import _winreg as winreg # used from python 2.0-2.6 |
---|
36 | except: |
---|
37 | import winreg # used from python 2.7 onwards |
---|
38 | net = winreg.OpenKey( |
---|
39 | winreg.HKEY_CURRENT_USER, |
---|
40 | "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings" |
---|
41 | ) |
---|
42 | n_subs, n_vals, last_mod = winreg.QueryInfoKey(net) |
---|
43 | subkeys = {} |
---|
44 | for i in range(n_vals): |
---|
45 | this_name, this_val, this_type = winreg.EnumValue(net, i) |
---|
46 | subkeys[this_name] = this_val |
---|
47 | if 'AutoConfigURL' in subkeys.keys() and len(subkeys['AutoConfigURL']) > 0: |
---|
48 | pac_files.append(subkeys['AutoConfigURL']) |
---|
49 | elif sys.platform == 'darwin': |
---|
50 | import plistlib |
---|
51 | sys_prefs = plistlib.readPlist( |
---|
52 | '/Library/Preferences/SystemConfiguration/preferences.plist') |
---|
53 | networks = sys_prefs['NetworkServices'] |
---|
54 | # loop through each possible network (e.g. Ethernet, Airport...) |
---|
55 | for network in networks.items(): |
---|
56 | # the first part is a long identifier |
---|
57 | net_key, network = network |
---|
58 | if 'ProxyAutoConfigURLString' in network['Proxies'].keys(): |
---|
59 | pac_files.append( |
---|
60 | network['Proxies']['ProxyAutoConfigURLString']) |
---|
61 | return list(set(pac_files)) # remove redundant ones |
---|
62 | |
---|
63 | def _parse_proxy_pac(self, pac_urls_list): |
---|
64 | ''' |
---|
65 | For every pac file url in pac_urls_list, it tryes to connect. |
---|
66 | If the connection is successful parses the file in search for |
---|
67 | http proxies. |
---|
68 | @param pac_urls_list: List with urls for the pac files |
---|
69 | @return: list with all found http proxies |
---|
70 | ''' |
---|
71 | proxy_url_list = [] |
---|
72 | for this_pac_url in pac_urls_list: |
---|
73 | logging.debug('Trying pac file (%s)...' % this_pac_url) |
---|
74 | try: |
---|
75 | response = urllib2.urlopen( |
---|
76 | this_pac_url, timeout=self.timeout) |
---|
77 | logging.debug('Succeeded (%s)...' % this_pac_url) |
---|
78 | except Exception: |
---|
79 | logging.debug('Failled (%s)...' % this_pac_url) |
---|
80 | continue |
---|
81 | pacStr = response.read() |
---|
82 | possProxies = re.findall( |
---|
83 | r"PROXY\s([^\s;,:]+:[0-9]{1,5})[^0-9]", pacStr + '\n') |
---|
84 | for thisPoss in possProxies: |
---|
85 | prox_url = 'http://' + thisPoss |
---|
86 | proxy_dic = {'http': prox_url} |
---|
87 | proxy_url_list.append(proxy_dic) |
---|
88 | return proxy_url_list |
---|
89 | |
---|
90 | def _set_proxy(self,proxy_dic=None): |
---|
91 | ''' |
---|
92 | Sets connection proxy. |
---|
93 | if proxy_dic is None get's teh proxy from the system. |
---|
94 | To disable autodetected proxy pass an empty dictionary: {} |
---|
95 | @param proxy_dic: format: {'http': 'http://www.example.com:3128/'} |
---|
96 | ''' |
---|
97 | if proxy_dic is None: |
---|
98 | # The default is to read the list of proxies from the environment variables <protocol>_proxy. |
---|
99 | # If no proxy environment variables are set, then in a Windows environment proxy settings are |
---|
100 | # obtained from the registry's Internet Settings section, and in a Mac OS X environment proxy |
---|
101 | # information is retrieved from the OS X System Configuration |
---|
102 | # Framework. |
---|
103 | proxy = urllib2.ProxyHandler() |
---|
104 | else: |
---|
105 | # If proxies is given, it must be a dictionary mapping protocol names to |
---|
106 | # URLs of proxies. |
---|
107 | proxy = urllib2.ProxyHandler(proxy_dic) |
---|
108 | opener = urllib2.build_opener(proxy) |
---|
109 | urllib2.install_opener(opener) |
---|
110 | |
---|
111 | |
---|
112 | |
---|
113 | |
---|
114 | def connect(self): |
---|
115 | ''' |
---|
116 | Performs the request and gets a response from self.url |
---|
117 | @return: response object from urllib2.urlopen |
---|
118 | ''' |
---|
119 | req = urllib2.Request(self.url) |
---|
120 | response = None |
---|
121 | try: |
---|
122 | logging.debug("Trying Direct connection to %s..."%self.url) |
---|
123 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
124 | except Exception, e: |
---|
125 | logging.debug("Failed!") |
---|
126 | logging.debug(e) |
---|
127 | try: |
---|
128 | logging.debug("Trying to use system proxy if it exists...") |
---|
129 | self._set_proxy() |
---|
130 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
131 | except Exception, e: |
---|
132 | logging.debug("Failed!") |
---|
133 | logging.debug(e) |
---|
134 | pac_urls = self._get_addresses_of_proxy_pac() |
---|
135 | proxy_urls = self._parse_proxy_pac(pac_urls) |
---|
136 | for proxy in proxy_urls: |
---|
137 | try: |
---|
138 | logging.debug("Trying to use the proxy %s found in proxy.pac configuration"%proxy) |
---|
139 | self._set_proxy(proxy) |
---|
140 | response = urllib2.urlopen(req, timeout=self.timeout) |
---|
141 | except Exception, e: |
---|
142 | logging.debug("Failed!") |
---|
143 | logging.debug(e) |
---|
144 | if response is not None: |
---|
145 | logging.debug("The connection to %s was successful."%self.url) |
---|
146 | else: |
---|
147 | logging.warning("Connection to %s failed..."%self.url) |
---|
148 | return response |
---|
149 | |
---|
150 | |
---|
151 | if __name__ == "__main__": |
---|
152 | from pprint import pprint |
---|
153 | c = Connection() |
---|
154 | response = c.connect() |
---|
155 | if response is not None: |
---|
156 | print 50 * '-' |
---|
157 | content = json.loads(response.read().strip()) |
---|
158 | pprint(content) |
---|