| 1 | """Python module for web browsing and scraping. |
|---|
| 2 | |
|---|
| 3 | Done: |
|---|
| 4 | - navigate to absolute and relative URLs |
|---|
| 5 | - follow links in page or region |
|---|
| 6 | - find first or all occurrences of string or RE in page or region |
|---|
| 7 | - find first, last, next, previous, or all tags with given name/attributes |
|---|
| 8 | - find first, last, next, previous, enclosing, or all elements with given |
|---|
| 9 | name/attributes/content |
|---|
| 10 | - set form fields |
|---|
| 11 | - submit forms |
|---|
| 12 | - strip tags from arbitrary strings of HTML |
|---|
| 13 | |
|---|
| 14 | Todo: |
|---|
| 15 | - cookie-handling is dumb (sends all cookies to all sites) |
|---|
| 16 | - handle CDATA and RCDATA marked sections |
|---|
| 17 | - support for submitting forms with file upload |
|---|
| 18 | - use Regions in striptags instead of duplicating work |
|---|
| 19 | - map of enders |
|---|
| 20 | """ |
|---|
| 21 | |
|---|
| 22 | __author__ = 'Ka-Ping Yee' |
|---|
| 23 | __date__ = '2005-03-29' |
|---|
| 24 | __version__ = '$Revision: 1.16 $' |
|---|
| 25 | |
|---|
| 26 | import os, socket, re, marshal, subprocess |
|---|
| 27 | from tempfile import gettempdir |
|---|
| 28 | from urlparse import urljoin, urlsplit |
|---|
| 29 | from urllib import urlencode |
|---|
| 30 | |
|---|
| 31 | def connect(server, port): |
|---|
| 32 | """Return a TCP socket connected to the given server and port.""" |
|---|
| 33 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
|---|
| 34 | sock.connect((server, port)) |
|---|
| 35 | return sock |
|---|
| 36 | |
|---|
| 37 | def receive(sock): |
|---|
| 38 | """Read all the data from a socket until it closes.""" |
|---|
| 39 | chunks = [] |
|---|
| 40 | while 1: |
|---|
| 41 | chunk = sock.recv(4096) |
|---|
| 42 | if chunk: chunks.append(chunk) |
|---|
| 43 | else: return ''.join(chunks) |
|---|
| 44 | |
|---|
| 45 | def request(host, method, path, headers, entity=None): |
|---|
| 46 | """Make an HTTP request and return (status, message, headers, document).""" |
|---|
| 47 | sock = connect(host, 80) |
|---|
| 48 | request = method + ' ' + path + ' HTTP/1.0\r\n' |
|---|
| 49 | for name in headers: |
|---|
| 50 | capname = '-'.join([part.capitalize() for part in name.split('-')]) |
|---|
| 51 | request += capname + ': ' + str(headers[name]) + '\r\n' |
|---|
| 52 | request += '\r\n' |
|---|
| 53 | if entity: |
|---|
| 54 | request += entity |
|---|
| 55 | sock.sendall(request) |
|---|
| 56 | data = receive(sock) |
|---|
| 57 | try: return splitreply(data) |
|---|
| 58 | except: return (0, '', {}, data) |
|---|
| 59 | |
|---|
| 60 | def splitreply(reply): |
|---|
| 61 | """Split an HTTP response into (status, message, headers, document).""" |
|---|
| 62 | if '\r\n\r\n' in reply: |
|---|
| 63 | head, document = reply.split('\r\n\r\n', 1) |
|---|
| 64 | else: |
|---|
| 65 | head, document = reply, '' |
|---|
| 66 | headers = [] |
|---|
| 67 | while True: |
|---|
| 68 | if '\r\n' in head: |
|---|
| 69 | response, head = head.split('\r\n', 1) |
|---|
| 70 | for line in head.split('\r\n'): |
|---|
| 71 | name, value = line.split(': ', 1) |
|---|
| 72 | headers.append((name.lower(), value)) |
|---|
| 73 | else: |
|---|
| 74 | response, head = head, '' |
|---|
| 75 | status = int(response.split()[1]) |
|---|
| 76 | message = ' '.join(response.split()[2:]) |
|---|
| 77 | if document.startswith('HTTP/1.') and '\r\n\r\n' in document: |
|---|
| 78 | head, document = document.split('\r\n\r\n', 1) |
|---|
| 79 | else: |
|---|
| 80 | return status, message, headers, document |
|---|
| 81 | |
|---|
| 82 | def shellquote(text): |
|---|
| 83 | """Quote a string literal for sh.""" |
|---|
| 84 | return "'" + text.replace("'", "'\\''") + "'" |
|---|
| 85 | |
|---|
| 86 | def curl(url, entity=None, follow=1, cookies=[], referrer=None): |
|---|
| 87 | """Invoke curl to perform an HTTP request.""" |
|---|
| 88 | command = ['curl', '-s', '-i'] |
|---|
| 89 | if referrer: |
|---|
| 90 | command += ['-e', referrer] |
|---|
| 91 | if entity: |
|---|
| 92 | if not isinstance(entity, str): |
|---|
| 93 | entity = urlencode(entity, doseq=1) |
|---|
| 94 | command += ['-d', entity] |
|---|
| 95 | if not follow: |
|---|
| 96 | command += ['-Z', '0'] |
|---|
| 97 | else: |
|---|
| 98 | command += ['-L'] |
|---|
| 99 | if cookies: |
|---|
| 100 | command += ['-b', '; '.join(cookies)] |
|---|
| 101 | command.append(url) |
|---|
| 102 | reply = subprocess.Popen(command, stdout=subprocess.PIPE).stdout.read() |
|---|
| 103 | return splitreply(reply) |
|---|
| 104 | |
|---|
| 105 | def fetch(url, entity=None, follow=1): |
|---|
| 106 | """Fetch one document in a one-shot session.""" |
|---|
| 107 | return Session().fetch(url, entity, follow) |
|---|
| 108 | |
|---|
| 109 | class ScrapeError(Exception): pass |
|---|
| 110 | class HTTPError(ScrapeError): pass |
|---|
| 111 | LAST_URL = object() |
|---|
| 112 | |
|---|
| 113 | class Session: |
|---|
| 114 | """A Web-browsing session. |
|---|
| 115 | |
|---|
| 116 | Exposed attributes: |
|---|
| 117 | |
|---|
| 118 | agent - set or get the User-Agent string |
|---|
| 119 | location - get the current (i.e. last successfully fetched) URL |
|---|
| 120 | status - get the status code of the last successful request |
|---|
| 121 | message - get the status message of the last successful request |
|---|
| 122 | headers - get the dictionary of headers from the last successful request |
|---|
| 123 | document - get the document returned by the last successful request |
|---|
| 124 | region - get a Region spanning the entire document |
|---|
| 125 | """ |
|---|
| 126 | |
|---|
| 127 | def __init__(self, agent=None): |
|---|
| 128 | self.cookies = [] |
|---|
| 129 | self.agent = agent |
|---|
| 130 | self.location = self.status = self.message = None |
|---|
| 131 | self.headers = self.document = self.region = None |
|---|
| 132 | self.history = [] |
|---|
| 133 | |
|---|
| 134 | def fetch(self, url, entity=None, follow=1, referrer=LAST_URL): |
|---|
| 135 | scheme, host, path, query, fragment = urlsplit(url) |
|---|
| 136 | if referrer is LAST_URL: |
|---|
| 137 | referrer = self.location |
|---|
| 138 | self.location = url |
|---|
| 139 | if scheme == 'https': |
|---|
| 140 | status, message, headers, document = \ |
|---|
| 141 | curl(url, entity, follow, self.cookies) |
|---|
| 142 | elif scheme == 'http': |
|---|
| 143 | if query: |
|---|
| 144 | path += '?' + query |
|---|
| 145 | headers = {} |
|---|
| 146 | headers['host'] = host |
|---|
| 147 | headers['accept'] = '*/*' |
|---|
| 148 | if referrer: |
|---|
| 149 | headers['referer'] = referrer |
|---|
| 150 | self.location = url |
|---|
| 151 | if self.agent: |
|---|
| 152 | headers['user-agent'] = self.agent |
|---|
| 153 | if self.cookies: |
|---|
| 154 | headers['cookie'] = '; '.join(self.cookies) |
|---|
| 155 | if entity: |
|---|
| 156 | if not isinstance(entity, str): |
|---|
| 157 | entity = urlencode(entity, doseq=1) |
|---|
| 158 | headers['content-type'] = 'application/x-www-form-urlencoded' |
|---|
| 159 | headers['content-length'] = len(entity) |
|---|
| 160 | method = entity and 'POST' or 'GET' |
|---|
| 161 | status, message, headers, document = \ |
|---|
| 162 | request(host, method, path, headers, entity) |
|---|
| 163 | else: |
|---|
| 164 | raise ValueError, scheme + ' not supported' |
|---|
| 165 | headerdict = {} |
|---|
| 166 | for name, value in headers: |
|---|
| 167 | if name == 'set-cookie': |
|---|
| 168 | cookie = value.split(';')[0] |
|---|
| 169 | if cookie not in self.cookies: |
|---|
| 170 | self.cookies.append(cookie) |
|---|
| 171 | else: |
|---|
| 172 | headerdict[name] = value |
|---|
| 173 | if follow and status in [301, 302] and 'location' in headerdict: |
|---|
| 174 | return self.fetch(urljoin(url, headerdict['location'])) |
|---|
| 175 | return status, message, headerdict, document |
|---|
| 176 | |
|---|
| 177 | def go(self, url, entity=None, follow=1, referrer=LAST_URL): |
|---|
| 178 | """Navigate to a given URL. If the URL is relative, it is resolved |
|---|
| 179 | with respect to the current location. If the document is successfully |
|---|
| 180 | fetched, return a Region spanning the entire document.""" |
|---|
| 181 | historyentry = (self.location, self.status, self.message, |
|---|
| 182 | self.headers, self.document, self.region) |
|---|
| 183 | if self.location: |
|---|
| 184 | url = urljoin(self.location, url) |
|---|
| 185 | results = self.fetch(url, entity, follow, referrer) |
|---|
| 186 | if results[0] == 200: |
|---|
| 187 | self.history.append(historyentry) |
|---|
| 188 | self.status, self.message, self.headers, self.document = results |
|---|
| 189 | self.region = Region(self.document) |
|---|
| 190 | return self.region |
|---|
| 191 | raise HTTPError(self.status, self.message) |
|---|
| 192 | |
|---|
| 193 | def back(self): |
|---|
| 194 | """Return to the previous page.""" |
|---|
| 195 | (self.location, self.status, self.message, |
|---|
| 196 | self.headers, self.document, self.region) = self.history.pop() |
|---|
| 197 | return self.location |
|---|
| 198 | |
|---|
| 199 | def follow(self, anchor, region=None): |
|---|
| 200 | """Follow the first link with the given anchor text. The anchor may |
|---|
| 201 | be given as a string or a compiled RE. If a region is given, the |
|---|
| 202 | link is sought within that region instead of the whole document.""" |
|---|
| 203 | link = (region or self.region).first('a', content=anchor) |
|---|
| 204 | if not link: |
|---|
| 205 | raise ScrapeError('link %r not found' % anchor) |
|---|
| 206 | if not link['href']: |
|---|
| 207 | raise ScrapeError('link %r has no href' % link) |
|---|
| 208 | return self.go(link['href']) |
|---|
| 209 | |
|---|
| 210 | def submit(self, form, button=None, **params): |
|---|
| 211 | """Submit a form, optionally by clicking a given button.""" |
|---|
| 212 | if form.tagname != 'form': |
|---|
| 213 | raise ScrapeError('%r is not a form' % form) |
|---|
| 214 | p = form.params |
|---|
| 215 | if button: |
|---|
| 216 | p[button['name']] = button['value'] |
|---|
| 217 | p.update(params) |
|---|
| 218 | method = form['method'].lower() or 'get' |
|---|
| 219 | if method == 'post': |
|---|
| 220 | return self.go(form['action'], p) |
|---|
| 221 | elif method == 'get': |
|---|
| 222 | return self.go(form['action'] + '?' + urlencode(p, doseq=1)) |
|---|
| 223 | else: |
|---|
| 224 | raise ScrapeError('unknown form method %r' % method) |
|---|
| 225 | |
|---|
| 226 | tagcontent_re = r'''(('[^']*'|"[^"]*"|--([^-]|-[^-])*--|-(?!-)|[^'">-])*)''' |
|---|
| 227 | |
|---|
| 228 | def tag_re(tagname_re): |
|---|
| 229 | return '<' + tagname_re + tagcontent_re + '>' |
|---|
| 230 | |
|---|
| 231 | anytag_re = tag_re(r'(\?|!\w*|/?[a-zA-Z_:][\w:.-]*)') |
|---|
| 232 | tagpat = re.compile(anytag_re) |
|---|
| 233 | |
|---|
| 234 | def htmldec(text): |
|---|
| 235 | """Decode HTML entities in the given text.""" |
|---|
| 236 | chunks = text.split('&#') |
|---|
| 237 | for i in range(1, len(chunks)): |
|---|
| 238 | number, rest = chunks[i].split(';', 1) |
|---|
| 239 | chunks[i] = chr(int(number)) + rest |
|---|
| 240 | text = ''.join(chunks) |
|---|
| 241 | text = text.replace('\xa0', ' ') |
|---|
| 242 | text = text.replace(' ', ' ') |
|---|
| 243 | text = text.replace('<', '<') |
|---|
| 244 | text = text.replace('>', '>') |
|---|
| 245 | text = text.replace('"', '"') |
|---|
| 246 | text = text.replace('&', '&') |
|---|
| 247 | return text |
|---|
| 248 | |
|---|
| 249 | def htmlenc(text): |
|---|
| 250 | """Use HTML entities to encode special characters in the given text.""" |
|---|
| 251 | text = text.replace('&', '&') |
|---|
| 252 | text = text.replace('"', '"') |
|---|
| 253 | text = text.replace('<', '<') |
|---|
| 254 | text = text.replace('>', '>') |
|---|
| 255 | return text |
|---|
| 256 | |
|---|
| 257 | def no_groups(re): |
|---|
| 258 | return re.replace('(', '(?:').replace('(?:?', '(?') |
|---|
| 259 | |
|---|
| 260 | tagsplitter = re.compile(no_groups(anytag_re)) |
|---|
| 261 | parasplitter = re.compile(no_groups(tag_re('(p|table|form)')), re.I) |
|---|
| 262 | linesplitter = re.compile(no_groups(tag_re('(div|br|tr)')), re.I) |
|---|
| 263 | scriptpat = re.compile(r'<script\b', re.I) |
|---|
| 264 | endscriptpat = re.compile(r'</script[^>]*>', re.I) |
|---|
| 265 | endcommentpat = re.compile(r'--\s*>') |
|---|
| 266 | |
|---|
| 267 | def striptags(text): |
|---|
| 268 | """Strip HTML tags from the given text, yielding line breaks for DIV, |
|---|
| 269 | BR, or TR tags and blank lines for P, TABLE, or FORM tags.""" |
|---|
| 270 | chunks = scriptpat.split(text) |
|---|
| 271 | for i in range(1, len(chunks)): |
|---|
| 272 | chunks[i] = endscriptpat.split(chunks[i], 1)[1] |
|---|
| 273 | text = ''.join(chunks) |
|---|
| 274 | chunks = text.split('<!') |
|---|
| 275 | for i in range(1, len(chunks)): |
|---|
| 276 | if chunks[i].split('>', 1)[0].find('--') >= 0: |
|---|
| 277 | chunks[i] = endcommentpat.split(chunks[i], 1)[1] |
|---|
| 278 | else: |
|---|
| 279 | chunks[i] = chunks[i].split('>', 1)[1] |
|---|
| 280 | text = ''.join(chunks) |
|---|
| 281 | |
|---|
| 282 | paragraphs = [] |
|---|
| 283 | for paragraph in parasplitter.split(text): |
|---|
| 284 | lines = [] |
|---|
| 285 | for line in linesplitter.split(paragraph): |
|---|
| 286 | line = ''.join(tagsplitter.split(line)) |
|---|
| 287 | line = htmldec(line) |
|---|
| 288 | line = ' '.join(line.split()) |
|---|
| 289 | lines.append(line) |
|---|
| 290 | paragraphs.append('\n'.join(lines)) |
|---|
| 291 | return re.sub('\n\n+', '\n\n', '\n\n'.join(paragraphs)).strip() |
|---|
| 292 | |
|---|
| 293 | attr_re = r'''\s*([\w:.-]+)(\s*=\s*('[^']*'|"[^"]*"|[^\s>]*))?''' |
|---|
| 294 | attrpat = re.compile(attr_re) |
|---|
| 295 | |
|---|
| 296 | def parseattrs(text): |
|---|
| 297 | """Turn a string of name=value pairs into an attribute dictionary.""" |
|---|
| 298 | attrs = {} |
|---|
| 299 | pos = 0 |
|---|
| 300 | while 1: |
|---|
| 301 | match = attrpat.search(text, pos) |
|---|
| 302 | if not match: break |
|---|
| 303 | pos = match.end() |
|---|
| 304 | name, value = match.group(1), match.group(3) or '' |
|---|
| 305 | if value[:1] in ["'", '"']: |
|---|
| 306 | value = value[1:-1] |
|---|
| 307 | attrs[name.lower()] = htmldec(value) |
|---|
| 308 | return attrs |
|---|
| 309 | |
|---|
| 310 | def matchcontent(specimen, desired): |
|---|
| 311 | if hasattr(desired, 'match'): |
|---|
| 312 | return desired.match(specimen) |
|---|
| 313 | elif callable(desired): |
|---|
| 314 | return desired(specimen) |
|---|
| 315 | else: |
|---|
| 316 | return specimen == desired |
|---|
| 317 | |
|---|
| 318 | def matchattrs(specimen, desired): |
|---|
| 319 | for name, value in desired.items(): |
|---|
| 320 | name = name.strip('_').replace('_', '-') |
|---|
| 321 | if not (name in specimen and matchcontent(specimen[name], value)): |
|---|
| 322 | return 0 |
|---|
| 323 | return 1 |
|---|
| 324 | |
|---|
| 325 | class Region: |
|---|
| 326 | """A Region object represents a contiguous region of a document together |
|---|
| 327 | with an associated HTML or XML tag and its attributes.""" |
|---|
| 328 | |
|---|
| 329 | def __init__(self, parent, start=0, end=None, starttag=None, endtag=None): |
|---|
| 330 | """Create a Region. The parent argument is a string or another |
|---|
| 331 | Region. The start and end arguments, if given, specify non-negative |
|---|
| 332 | indices into the original string (not into a parent subregion).""" |
|---|
| 333 | if isinstance(parent, Region): |
|---|
| 334 | self.document = parent.document |
|---|
| 335 | self.tags = parent.tags |
|---|
| 336 | else: |
|---|
| 337 | self.document = parent |
|---|
| 338 | self.tags = self.scantags(self.document) |
|---|
| 339 | if end is None: |
|---|
| 340 | end = len(self.document) |
|---|
| 341 | self.start, self.end = start, end |
|---|
| 342 | self.tagname, self.attrs = None, {} |
|---|
| 343 | |
|---|
| 344 | # If only starttag is specified, this Region is a tag. |
|---|
| 345 | # If starttag and endtag are specified, this Region is an element. |
|---|
| 346 | self.starttag, self.endtag = starttag, endtag |
|---|
| 347 | if starttag is not None: |
|---|
| 348 | self.start, self.end, self.tagname, self.attrs = self.tags[starttag] |
|---|
| 349 | if endtag is not None: |
|---|
| 350 | self.start, self.end = self.tags[starttag][1], self.tags[endtag][0] |
|---|
| 351 | |
|---|
| 352 | # Find the minimum and maximum indices of tags within this Region. |
|---|
| 353 | if starttag and endtag: |
|---|
| 354 | self.tagmin, self.tagmax = starttag + 1, endtag - 1 |
|---|
| 355 | else: |
|---|
| 356 | self.tagmin, self.tagmax = len(self.tags), -1 |
|---|
| 357 | for i, (start, end, tagname, attrs) in enumerate(self.tags): |
|---|
| 358 | if start >= self.start and i < self.tagmin: |
|---|
| 359 | self.tagmin = i |
|---|
| 360 | if end <= self.end and i > self.tagmax: |
|---|
| 361 | self.tagmax = i |
|---|
| 362 | |
|---|
| 363 | def __repr__(self): |
|---|
| 364 | if self.tagname: |
|---|
| 365 | attrs = ''.join([' %s=%r' % item for item in self.attrs.items()]) |
|---|
| 366 | return '<Region %d:%d %s%s>' % ( |
|---|
| 367 | self.start, self.end, self.tagname, attrs) |
|---|
| 368 | else: |
|---|
| 369 | return '<Region %d:%d>' % (self.start, self.end) |
|---|
| 370 | |
|---|
| 371 | # Utilities that operate on the array of scanned tags. |
|---|
| 372 | def scantags(self, document): |
|---|
| 373 | """Generate a list of all the tags in a document.""" |
|---|
| 374 | tags = [] |
|---|
| 375 | pos = 0 |
|---|
| 376 | while 1: |
|---|
| 377 | match = tagpat.search(document, pos) |
|---|
| 378 | if not match: break |
|---|
| 379 | start, end = match.span() |
|---|
| 380 | tagname = match.group(1).lower() |
|---|
| 381 | attrs = match.group(2) |
|---|
| 382 | tags.append([start, end, tagname, attrs]) |
|---|
| 383 | if tagname == 'script': |
|---|
| 384 | match = endscriptpat.search(document, end) |
|---|
| 385 | if not match: break |
|---|
| 386 | start, end = match.span() |
|---|
| 387 | tags.append([start, end, '/' + tagname, '']) |
|---|
| 388 | pos = end |
|---|
| 389 | return tags |
|---|
| 390 | |
|---|
| 391 | def matchtag(self, i, tagname, attrs): |
|---|
| 392 | """Return 1 if the ith tag matches the given tagname and attributes.""" |
|---|
| 393 | itagname, iattrs = self.tags[i][2], self.tags[i][3] |
|---|
| 394 | if itagname[:1] not in ['', '?', '!', '/']: |
|---|
| 395 | if itagname == tagname or tagname is None: |
|---|
| 396 | if isinstance(iattrs, str): |
|---|
| 397 | self.tags[i][3] = iattrs = parseattrs(iattrs) |
|---|
| 398 | return matchattrs(iattrs, attrs) |
|---|
| 399 | |
|---|
| 400 | def findendtag(self, starttag, outside=0): |
|---|
| 401 | """Find the index of the matching end tag for the given start tag. |
|---|
| 402 | If outside is 0, look for the end tag within the current region; |
|---|
| 403 | if outside is 1, look beyond the end of the current region.""" |
|---|
| 404 | tagname = self.tags[starttag][2] |
|---|
| 405 | depth = 1 |
|---|
| 406 | for i in range(starttag + 1, len(self.tags)): |
|---|
| 407 | if self.tags[i][2] == tagname: |
|---|
| 408 | depth += 1 |
|---|
| 409 | if self.tags[i][2] == '/' + tagname: |
|---|
| 410 | depth -= 1 |
|---|
| 411 | if depth == 0: |
|---|
| 412 | if not outside and i <= self.tagmax: |
|---|
| 413 | return i |
|---|
| 414 | if outside and i > self.tagmax: |
|---|
| 415 | return i |
|---|
| 416 | break |
|---|
| 417 | |
|---|
| 418 | def matchelement(self, starttag, content=None, outside=0): |
|---|
| 419 | """If the element with the given start tag matches the given content, |
|---|
| 420 | return the index of the matching end tag. See findendtag() for the |
|---|
| 421 | meaning of the outside flag.""" |
|---|
| 422 | endtag = self.findendtag(starttag, outside) |
|---|
| 423 | if endtag is not None: |
|---|
| 424 | start, end = self.tags[starttag][1], self.tags[endtag][0] |
|---|
| 425 | stripped = striptags(self.document[start:end]) |
|---|
| 426 | if content is None or matchcontent(stripped, content): |
|---|
| 427 | return endtag |
|---|
| 428 | |
|---|
| 429 | # Provide the "content" and "text" attributes to access the contents. |
|---|
| 430 | content = property(lambda self: self.document[self.start:self.end]) |
|---|
| 431 | text = property(lambda self: striptags(self.content)) |
|---|
| 432 | |
|---|
| 433 | def getparams(self): |
|---|
| 434 | """Get a dictionary of default values for all the form parameters.""" |
|---|
| 435 | if self.tagname == 'form': |
|---|
| 436 | params = {} |
|---|
| 437 | for input in self.alltags('input'): |
|---|
| 438 | if 'disabled' not in input: |
|---|
| 439 | type = input['type'].lower() |
|---|
| 440 | if type in ['text', 'password', 'hidden'] or ( |
|---|
| 441 | type in ['checkbox', 'radio'] and 'checked' in input): |
|---|
| 442 | params[input['name']] = input['value'] |
|---|
| 443 | for select in self.all('select'): |
|---|
| 444 | if 'disabled' not in select: |
|---|
| 445 | selections = [option['value'] |
|---|
| 446 | for option in select.alltags('option') |
|---|
| 447 | if 'selected' in option] |
|---|
| 448 | if 'multiple' in select: |
|---|
| 449 | params[select['name']] = selections |
|---|
| 450 | elif selections: |
|---|
| 451 | params[select['name']] = selections[0] |
|---|
| 452 | for textarea in self.all('textarea'): |
|---|
| 453 | if 'disabled' not in textarea: |
|---|
| 454 | params[textarea['name']] = textarea.content |
|---|
| 455 | return params |
|---|
| 456 | |
|---|
| 457 | def getbuttons(self): |
|---|
| 458 | """Get a list of all the form submission buttons.""" |
|---|
| 459 | if self.tagname == 'form': |
|---|
| 460 | return [tag for tag in self.alltags('input') |
|---|
| 461 | if tag['type'].lower() in ['submit', 'image'] |
|---|
| 462 | ] + [tag for tag in self.alltags('button') |
|---|
| 463 | if tag['type'].lower() in ['submit', '']] |
|---|
| 464 | |
|---|
| 465 | params = property(getparams) |
|---|
| 466 | buttons = property(getbuttons) |
|---|
| 467 | |
|---|
| 468 | # Provide a dictionary-like interface to the tag attributes. |
|---|
| 469 | def __contains__(self, name): |
|---|
| 470 | return name in self.attrs |
|---|
| 471 | |
|---|
| 472 | def __getitem__(self, name): |
|---|
| 473 | return self.attrs.get(name, '') |
|---|
| 474 | |
|---|
| 475 | # Provide subregions by slicing. |
|---|
| 476 | def __getslice__(self, start, end): |
|---|
| 477 | start += (start < 0) and self.end or self.start |
|---|
| 478 | end += (end < 0) and self.end or self.start |
|---|
| 479 | return Region(self, start, end) |
|---|
| 480 | |
|---|
| 481 | # Search for text. |
|---|
| 482 | def find(self, target, group=0): |
|---|
| 483 | """Search this Region for a string or a compiled RE and return a |
|---|
| 484 | Region representing the match. The optional group argument specifies |
|---|
| 485 | which grouped subexpression should be returned as the match.""" |
|---|
| 486 | if hasattr(target, 'search'): |
|---|
| 487 | match = target.search(self.content) |
|---|
| 488 | if match: |
|---|
| 489 | return self[match.start(group):match.end(group)] |
|---|
| 490 | else: |
|---|
| 491 | start = self.content.find(target) |
|---|
| 492 | if start > -1: |
|---|
| 493 | return self[start:start+len(target)] |
|---|
| 494 | |
|---|
| 495 | def findall(self, target, group=0): |
|---|
| 496 | """Search this Region for a string or a compiled RE and return a |
|---|
| 497 | sequence of Regions representing all the matches.""" |
|---|
| 498 | pos = 0 |
|---|
| 499 | content = self.content |
|---|
| 500 | matches = [] |
|---|
| 501 | if hasattr(target, 'search'): |
|---|
| 502 | while 1: |
|---|
| 503 | match = target.search(content, pos) |
|---|
| 504 | if not match: |
|---|
| 505 | break |
|---|
| 506 | start, pos = match.span(group) |
|---|
| 507 | matches.append(self[start:pos]) |
|---|
| 508 | else: |
|---|
| 509 | while 1: |
|---|
| 510 | start = content.find(target, pos) |
|---|
| 511 | if start < 0: |
|---|
| 512 | break |
|---|
| 513 | pos = start + len(target) |
|---|
| 514 | matches.append(self[start:pos]) |
|---|
| 515 | return matches |
|---|
| 516 | |
|---|
| 517 | # Search for tags. |
|---|
| 518 | def firsttag(self, tagname=None, **attrs): |
|---|
| 519 | """Return the Region for the first tag entirely within this Region |
|---|
| 520 | with the given tag name and attributes.""" |
|---|
| 521 | for i in range(self.tagmin, self.tagmax + 1): |
|---|
| 522 | if self.matchtag(i, tagname, attrs): |
|---|
| 523 | return Region(self, 0, 0, i) |
|---|
| 524 | |
|---|
| 525 | def lasttag(self, tagname=None, **attrs): |
|---|
| 526 | """Return the Region for the last tag entirely within this Region |
|---|
| 527 | with the given tag name and attributes.""" |
|---|
| 528 | for i in range(self.tagmax, self.tagmin - 1, -1): |
|---|
| 529 | if self.matchtag(i, tagname, attrs): |
|---|
| 530 | return Region(self, 0, 0, i) |
|---|
| 531 | |
|---|
| 532 | def alltags(self, tagname=None, **attrs): |
|---|
| 533 | """Return a list of Regions for all the tags entirely within this |
|---|
| 534 | Region with the given tag name and attributes.""" |
|---|
| 535 | tags = [] |
|---|
| 536 | for i in range(self.tagmin, self.tagmax + 1): |
|---|
| 537 | if self.matchtag(i, tagname, attrs): |
|---|
| 538 | tags.append(Region(self, 0, 0, i)) |
|---|
| 539 | return tags |
|---|
| 540 | |
|---|
| 541 | def nexttag(self, tagname=None, **attrs): |
|---|
| 542 | """Return the Region for the nearest tag after the end of this Region |
|---|
| 543 | with the given tag name and attributes.""" |
|---|
| 544 | return Region(self, self.end).firsttag(tagname, **attrs) |
|---|
| 545 | |
|---|
| 546 | def previoustag(self, tagname=None, **attrs): |
|---|
| 547 | """Return the Region for the nearest tag before the start of this |
|---|
| 548 | Region with the given tag name and attributes.""" |
|---|
| 549 | return Region(self, 0, self.start).lasttag(tagname, **attrs) |
|---|
| 550 | |
|---|
| 551 | # Search for elements. |
|---|
| 552 | def first(self, tagname=None, content=None, **attrs): |
|---|
| 553 | """Return the Region for the first properly balanced element entirely |
|---|
| 554 | within this Region with the given tag name, content, and attributes. |
|---|
| 555 | The element content is passed through striptags(). If the content |
|---|
| 556 | argument has a match() method, the stripped content is passed into |
|---|
| 557 | this method; otherwise it is compared directly as a string.""" |
|---|
| 558 | for starttag in range(self.tagmin, self.tagmax + 1): |
|---|
| 559 | if self.matchtag(starttag, tagname, attrs): |
|---|
| 560 | endtag = self.matchelement(starttag, content) |
|---|
| 561 | if endtag is not None: |
|---|
| 562 | return Region(self, 0, 0, starttag, endtag) |
|---|
| 563 | |
|---|
| 564 | def last(self, tagname=None, content=None, **attrs): |
|---|
| 565 | """Return the Region for the last properly balanced element entirely |
|---|
| 566 | within this Region with the given tag name, content, and attributes.""" |
|---|
| 567 | for starttag in range(self.tagmax, self.tagmin - 1, -1): |
|---|
| 568 | if self.matchtag(starttag, tagname, attrs): |
|---|
| 569 | endtag = self.matchelement(starttag, content) |
|---|
| 570 | if endtag is not None: |
|---|
| 571 | return Region(self, 0, 0, starttag, endtag) |
|---|
| 572 | |
|---|
| 573 | def all(self, tagname=None, content=None, **attrs): |
|---|
| 574 | """Return Regions for all non-overlapping balanced elements entirely |
|---|
| 575 | within this Region with the given tag name, content, and attributes.""" |
|---|
| 576 | elements = [] |
|---|
| 577 | starttag = self.tagmin |
|---|
| 578 | while starttag <= self.tagmax: |
|---|
| 579 | if self.matchtag(starttag, tagname, attrs): |
|---|
| 580 | endtag = self.matchelement(starttag, content) |
|---|
| 581 | if endtag is not None: |
|---|
| 582 | elements.append(Region(self, 0, 0, starttag, endtag)) |
|---|
| 583 | starttag = endtag |
|---|
| 584 | starttag += 1 |
|---|
| 585 | return elements |
|---|
| 586 | |
|---|
| 587 | def next(self, tagname=None, content=None, **attrs): |
|---|
| 588 | """Return the Region for the nearest balanced element after the end of |
|---|
| 589 | this Region with the given tag name, content, and attributes.""" |
|---|
| 590 | return Region(self, self.end).first(tagname, content, **attrs) |
|---|
| 591 | |
|---|
| 592 | def previous(self, tagname=None, content=None, **attrs): |
|---|
| 593 | """Return the Region for the nearest balanced element before the start |
|---|
| 594 | of this Region with the given tag name, content, and attributes.""" |
|---|
| 595 | return Region(self, 0, self.start).last(tagname, content, **attrs) |
|---|
| 596 | |
|---|
| 597 | def enclosing(self, tagname=None, content=None, **attrs): |
|---|
| 598 | """Return the Region for the nearest balanced element that encloses |
|---|
| 599 | this Region with the given tag name, content, and attributes.""" |
|---|
| 600 | if self.starttag and self.endtag: # skip our own start tag |
|---|
| 601 | laststarttag = self.starttag - 1 |
|---|
| 602 | else: |
|---|
| 603 | laststarttag = self.tagmin - 1 |
|---|
| 604 | for starttag in range(laststarttag, -1, -1): |
|---|
| 605 | if self.matchtag(starttag, tagname, attrs): |
|---|
| 606 | endtag = self.matchelement(starttag, content, outside=1) |
|---|
| 607 | if endtag is not None: |
|---|
| 608 | return Region(self, 0, 0, starttag, endtag) |
|---|
| 609 | |
|---|
| 610 | def read(path): |
|---|
| 611 | """Read and return the entire contents of the file at the given path.""" |
|---|
| 612 | return open(path).read() |
|---|
| 613 | |
|---|
| 614 | def write(path, text): |
|---|
| 615 | """Write the given text to a file at the given path.""" |
|---|
| 616 | file = open(path, 'w') |
|---|
| 617 | file.write(text) |
|---|
| 618 | file.close() |
|---|
| 619 | |
|---|
| 620 | def load(path): |
|---|
| 621 | """Return the deserialized contents of the file at the given path.""" |
|---|
| 622 | return marshal.load(open(path)) |
|---|
| 623 | |
|---|
| 624 | def dump(path, data): |
|---|
| 625 | """Serialize the given data and write it to a file at the given path.""" |
|---|
| 626 | file = open(path, 'w') |
|---|
| 627 | marshal.dump(data, file) |
|---|
| 628 | file.close() |
|---|
| 629 | |
|---|
| 630 | def getnumber(text): |
|---|
| 631 | """Find and parse a floating-point or integer number in the given text, |
|---|
| 632 | ignoring commas, percentage signs, and non-numeric words.""" |
|---|
| 633 | for word in striptags(text).replace(',', '').replace('%', '').split(): |
|---|
| 634 | try: return int(word) |
|---|
| 635 | except: |
|---|
| 636 | try: return float(word) |
|---|
| 637 | except: continue |
|---|
