HTTP/1.1 module 
Author Message
 HTTP/1.1 module

Hi all,

Here is a new httplib module that can do HTTP/1.1, including persistent
connections (actual, and programmatically simulated), "chunked" transfer
encoding, simple request retries, and full backwards compatibility with
the standard httplib.

All comments are welcome (bugs, design suggestions, enhancements, etc).


Greg Stein, http://www.*-*-*.com/

[ 10K ]
# HTTP/1.1 client library

import socket
import string
import mimetools

error = __name__ + '.error'


class HTTPConnection:

  _http_vsn = 11
  _http_vsn_str = 'HTTP/1.1'

  def __init__(self, host, port=None):
    self.sock = None
    self.response = None
    self._set_hostport(host, port)

  def _set_hostport(self, host, port):
    if port is None:
      i = string.find(host, ':')
      if i >= 0:
        port = int(host[i+1:])
        host = host[:i]
        port = HTTP_PORT = host
    self.port = port

  def connect(self):
    """Connect to the host and port specified in __init__."""
    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.sock.connect(, self.port)

  def close(self):
    """Close the connection to the HTTP server."""
    if self.sock:
      self.sock.close() # close it manually... there may be other refs
      self.sock = None
    if self.response:
      self.response = None

  def send(self, str):
    """Send `str' to the server."""
    if self.sock is None:

    # send the data to the server. if we get a broken pipe, then close
    # the socket. we want to reconnect when somebody tries to send again.
    # NOTE: we DO propagate the error, though, because we cannot simply
    #       ignore the error... the caller will know if they can retry.
    except socket.error, v:
      if v[0] == 32:    # Broken pipe

  def putrequest(self, method, url):
    """Send a request to the server.

    `method' specifies an HTTP request method, e.g. 'GET'.
    `url' specifies the object being requested, e.g.
    if self.response is not None:
      if not self.response.isclosed():
        ### implies half-duplex!
        raise error, 'prior response has not been fully handled'
      self.response = None

    if not url:
      url = '/'
    str = '%s %s %s\r\n' % (method, url, self._http_vsn_str)

    except socket.error, v:
      if v[0] != 32:    # Broken pipe
      # try one more time (the socket was closed; this will reopen)


    if self._http_vsn == 11:
      # Issue some standard headers for better HTTP/1.1 compliance

      # note: we are assuming that clients will not attempt to set these
      #       headers since *this* library must deal with the consequences.
      #       this also means that when the supporting libraries are
      #       updated to recognize other forms, then this code should be
      #       changed (removed or updated).

      # we only want a Content-Encoding of "identity" since we don't
      # support encodings such as x-gzip or x-deflate.
      self.putheader('Accept-Encoding', 'identity')

      # we can accept "chunked" Transfer-Encodings, but no others
      # NOTE: no TE header implies *only* "chunked"
      #self.putheader('TE', 'chunked')

      # if TE is supplied in the header, then it must appear in a
      # Connection header.
      #self.putheader('Connection', 'TE')

      # For HTTP/1.0, the server will assume "not chunked"

  def putheader(self, header, value):
    """Send a request header line to the server.

    For example: h.putheader('Accept', 'text/html')
    str = '%s: %s\r\n' % (header, value)

  def endheaders(self):
    """Indicate that the last header line has been sent to the server."""


  def request(self, method, url, body=None, headers={}):
    """Send a complete request to the server."""

      self._send_request(method, url, body, headers)
    except socket.error, v:
      if v[0] != 32:    # Broken pipe
      # try one more time
      self._send_request(method, url, body, headers)

  def _send_request(self, method, url, body, headers):
    self.putrequest(method, url)

    if body:
      self.putheader('Content-Length', str(len(body)))
    for hdr, value in headers.items():
      self.putheader(hdr, value)

    if body:

  def getreply(self):
    """Get a reply from the server.

    Returns a tuple consisting of:
    - server response code (e.g. '200' if all goes well)
    - server response string corresponding to response code
    - any RFC822 headers in the response from the server

    file = self.sock.makefile('rb')
    line = file.readline()
      [ver, code, msg] = string.split(line, None, 2)
    except ValueError:
        [ver, code] = string.split(line, None, 1)
        msg = ""
      except ValueError:
        return -1, line, file
    if ver[:5] != 'HTTP/':
      return -1, line, file
    errcode = int(code)
    errmsg = string.strip(msg)
    response = HTTPResponse(file)
    if response.will_close:
      # this effectively passes the connection to the response
      # remember this, so we can tell when it is complete
      self.response = response
    return errcode, errmsg, response

class HTTPResponse(mimetools.Message):
  def __init__(self, fp):
    mimetools.Message.__init__(self, fp, 0)

    # are we using the chunked-style of transfer encoding?
    tr_enc = self.getheader('transfer-encoding')
    if tr_enc:
      if string.lower(tr_enc) != 'chunked':
        raise error, 'unknown transfer-encoding'
      self.chunked = 1
      self.chunk_left = None
      self.chunked = 0

    # will the connection close at the end of the response?
    conn = self.getheader('connection')
    if conn:
      self.will_close = string.find(string.lower(conn), 'close') != -1
      self.will_close = 0

    # do we have a Content-Length?
    length = self.getheader('content-length')
    if length:
      self.length = int(length)
      self.length = None

    # if the connection remains open, and we aren't using chunked, and
    # a content-length was not provided, then barf.
    if not self.will_close and \
       not self.chunked and \
       length is None:
      raise error, 'protocol error: Content-Length is required'

  def close(self):
    if self.fp:
      self.fp = None

  def isclosed(self):
    # NOTE: it is possible that we will not ever call self.close(). This
    #       case occurs when will_close is TRUE, length is None, and we
    #       read up to the last byte, but NOT past it.
    # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
    #          called, meaning self.isclosed() is meaningful.
    return self.fp is None

  def read(self, amt=None):
    if self.fp is None:
      return ''

    if self.chunked:
      chunk_left = self.chunk_left
      value = ''
      while 1:
        if chunk_left is None:
          line = self.fp.readline()
          i = string.find(line, ';')
          if i >= 0:
            line = line[:i]     # strip chunk-extensions
          chunk_left = string.atoi(line, 16)
          if chunk_left == 0:
        if amt is None:
          value = value +
        elif amt < chunk_left:
          value = value +
          self.chunk_left = chunk_left - amt
          return value
        elif amt == chunk_left:
          value = value +
       # toss the CRLF at the end of the chunk
          self.chunk_left = None
          return value
          value = value +
          amt = amt - chunk_left

        # we read the whole chunk, get another         # toss the CRLF at the end of the chunk
        chunk_left = None

      # read and discard trailer up to the CRLF terminator
      ### note: we shouldn't have any trailers!
      while 1:
        line = self.fp.readline()
        if line == '\r\n':

      # we read everything; close the "file"

      return value

    elif amt is None:
      # unbounded read
      if self.will_close:
        s =
        s =
      self.close()      # we read everything
      return s

    if self.length is not None:
      if amt > self.length:
        # clip the read to the "end of response"
        amt = self.length
      self.length = self.length - amt

    s =

    # close our "file" if we know we should
    ### I'm not sure about the len(s) < amt part; we should be safe because
    ### we shouldn't be using non-blocking sockets
    if self.length == 0 or len(s) < amt:

    return s

class HTTP(HTTPConnection):
  "Compatibility class with from 1.5."

  _http_vsn = 10
  _http_vsn_str = 'HTTP/1.0'

  def __init__(self, host='', port=None):
    "Provide a default host, since the superclass requires one."

    # Note that we may pass an empty string as the host; this will throw
    # an error when we attempt to connect. Presumably, the client code
    # will call connect before then, with a proper host.
    HTTPConnection.__init__(self, host, port)

  def connect(self, host, port=None):
    "Accept arguments to set the host/port, since the superclass doesn't."

    self._set_hostport(host, port)

  def set_debuglevel(self, debuglevel):
    "The class no longer supports the debuglevel."

  def getfile(self):
    "Provide a getfile, since the superclass' use of HTTP/1.1 prevents it."
    return self.file

  def putheader(self, header, *values):
    "The superclass allows only one value argument."
    HTTPConnection.putheader(self, header, string.joinfields(values,'\r\n\t'))

  def getreply(self):
    "Compensate for an instance attribute shuffling."
    errcode, errmsg, response = HTTPConnection.getreply(self)
    if errcode == -1:

read more »

Mon, 23 Jul 2001 03:00:00 GMT  
 HTTP/1.1 module
ah! good find.

I think the right way to fix that is to tweak HTTP.connect:

def connect(self, host=None, port=None):
  if host is not None:
    self._set_hostpost(host, port)

I just tried that out and it seems to works fine.



> Hi!

> I've just tried your new moduile, but it fails
> when called from urllib. The reason is that
> your compatibility class redefines "connect"
> but the superclass calls "self.connect" and
> the arg count is wrong:

>         : s tmp ; python
>         Python 1.5.1 (#11, Jul  7 1998, 16:34:26)  [GCC] on sunos5
>         Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam
>         >>> import urllib
>         >>> x=urllib.urlopen('')
>         Traceback (innermost last):
>           File "<stdin>", line 1, in ?
>           File "/local/usr/lib/python1.5/", line 59, in urlopen
>             return
>           File "/local/usr/lib/python1.5/", line 159, in open
>             return getattr(self, name)(url)
>           File "/local/usr/lib/python1.5/", line 253, in open_http
>             h.putrequest('GET', selector)
>           File "", line 85, in putrequest
>             self.send(str)
>           File "", line 53, in send
>             self.connect()
>         TypeError: not enough arguments; expected 2, got 1

Greg Stein,

Tue, 24 Jul 2001 03:00:00 GMT  
 [ 2 post ] 

 Relevant Pages 

1. HTTP 1.0 and HTTP 1.1 packages

2. HTTP 1.1 support in BaseHTTPServer [patch 430706]

3. HTTP/1.1 server in Python

4. HTTP 1.1 anyone?

5. httplib supporting HTTP/1.1 [Q]

6. HTTP/1.1 support in 1.5?

7. HTTP/1.1 BaseHTTPServer

8. python 1.4 and HTTP/1.1

9. urllib HTTP/1.1 patch

10. fixed for HTTP/1.1?

11. FAQ: Patch for HTTP/1.1 in httplib...

12. bug in httplib for HTTP/1.1 responses


Powered by phpBB® Forum Software