/* HTTP request/response handling Copyright (C) 1999-2003, Joe Orton This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */ /* This is the HTTP client request/response implementation. * The goal of this code is to be modular and simple. */ #include "config.h" #include #include #ifdef __EMX__ #include #endif #ifdef HAVE_LIMITS_H #include /* for UINT_MAX etc */ #endif #include #include #include #ifdef HAVE_STRING_H #include #endif #ifdef HAVE_STRINGS_H #include #endif #ifdef HAVE_STDLIB_H #include #endif #ifdef HAVE_UNISTD_H #include #endif #include "ne_i18n.h" #include "ne_alloc.h" #include "ne_request.h" #include "ne_string.h" /* for ne_buffer */ #include "ne_utils.h" #include "ne_socket.h" #include "ne_uri.h" #include "ne_private.h" #define HTTP_EXPECT_TIMEOUT 15 /* 100-continue only used if size > HTTP_EXPECT_MINSIZ */ #define HTTP_EXPECT_MINSIZE 1024 /* with thanks to Jim Blandy; this macro simplified loads of code. */ #define HTTP_ERR(x) do { int _ret = (x); if (_ret != NE_OK) return _ret; } while (0) #define SOCK_ERR(req, op, msg) do { ssize_t sret = (op); \ if (sret < 0) return aborted(req, msg, sret); } while (0) /* This is called with each of the headers in the response */ struct header_handler { char *name; ne_header_handler handler; void *userdata; struct header_handler *next; }; /* TODO: could unify these all into a generic callback list */ struct body_reader { ne_block_reader handler; ne_accept_response accept_response; unsigned int use:1; void *userdata; struct body_reader *next; }; struct ne_request_s { char *method, *uri; /* method and Request-URI */ ne_buffer *headers; /* request headers */ /* Request body. */ ne_provide_body body_cb; void *body_ud; /* Comes from either an fd or a buffer. */ union { int fd; struct { const char *buffer, *pnt; size_t left; } buf; } body; size_t body_size, body_progress; /* temporary store for response lines. */ char respbuf[BUFSIZ]; /**** Response ***/ /* The transfer encoding types */ struct ne_response { int length; /* Response entity-body content-length */ size_t left; /* Bytes left to read */ size_t chunk_left; /* Bytes of chunk left to read */ size_t total; /* total bytes read so far. */ /* how the message length is detemined: */ enum { R_TILLEOF = 0, /* read till eof */ R_NO_BODY, /* implicitly no body (HEAD, 204, 304) */ R_CHUNKED, /* using chunked transfer-encoding */ R_CLENGTH /* using given content-length */ } mode; } resp; /* List of callbacks which are passed response headers */ struct header_handler *header_catchers; struct hook *private; /* We store response header handlers in a hash table. The hash is * derived from the header name in lower case. */ /* FIXME: this comment is WAY out of date, and no longer in any * way true. */ /* 53 is magic, of course. For a standard ne_get() (with * redirects), 9 header handlers are defined. Two of these are * for Content-Length (which is a bug, and should be fixed * really). Ignoring that hash clash, the 8 *different* handlers * all hash uniquely into the hash table of size 53. */ #define HH_HASHSIZE 53 struct header_handler *header_handlers[HH_HASHSIZE]; /* List of callbacks which are passed response body blocks */ struct body_reader *body_readers; /*** Miscellaneous ***/ unsigned int method_is_head:1; unsigned int use_expect100:1; unsigned int can_persist:1; ne_session *session; ne_status status; }; static int open_connection(ne_request *req); /* The iterative step used to produce the hash value. This is DJB's * magic "*33" hash function. Ralf Engelschall has done some amazing * statistical analysis to show that *33 really is a good hash * function: check the new-httpd list archives, or his 'str' library * source code, for the details. * * TODO: due to limited range of characters used in header names, * could maybe get a better hash function to use? */ #define HH_ITERATE(hash, ch) (((hash)*33 + (ch)) % HH_HASHSIZE); /* Returns hash value for header 'name', converting it to lower-case * in-place. */ static inline unsigned int hash_and_lower(char *name) { char *pnt; unsigned int hash = 0; for (pnt = name; *pnt != '\0'; pnt++) { *pnt = tolower(*pnt); hash = HH_ITERATE(hash,*pnt); } return hash; } /* Abort a request due to an non-recoverable HTTP protocol error, * whilst doing 'doing'. 'code', if non-zero, is the socket error * code, NE_SOCK_*, or if zero, is ignored. */ static int aborted(ne_request *req, const char *doing, ssize_t code) { ne_session *sess = req->session; int ret = NE_ERROR; NE_DEBUG(NE_DBG_HTTP, "Aborted request (%" NE_FMT_SSIZE_T "): %s\n", code, doing); switch(code) { case NE_SOCK_CLOSED: if (sess->use_proxy) { ne_set_error(sess, _("%s: connection was closed by proxy server."), doing); } else { ne_set_error(sess, _("%s: connection was closed by server."), doing); } break; case NE_SOCK_TIMEOUT: ne_set_error(sess, _("%s: connection timed out."), doing); ret = NE_TIMEOUT; break; case NE_SOCK_ERROR: case NE_SOCK_RESET: case NE_SOCK_TRUNC: ne_set_error(sess, "%s: %s", doing, ne_sock_error(sess->socket)); break; case 0: ne_set_error(sess, "%s", doing); break; } ne_close_connection(sess); return ret; } static void notify_status(ne_session *sess, ne_conn_status status, const char *info) { if (sess->notify_cb) { sess->notify_cb(sess->notify_ud, status, info); } } void ne_duplicate_header(void *userdata, const char *value) { char **location = userdata; *location = ne_strdup(value); } void ne_handle_numeric_header(void *userdata, const char *value) { int *location = userdata; *location = atoi(value); } static void *get_private(const struct hook *hk, const char *id) { for (; hk != NULL; hk = hk->next) if (strcmp(hk->id, id) == 0) return hk->userdata; return NULL; } void *ne_get_request_private(ne_request *req, const char *id) { return get_private(req->private, id); } void *ne_get_session_private(ne_session *sess, const char *id) { return get_private(sess->private, id); } typedef void (*void_fn)(void); #define ADD_HOOK(hooks, fn, ud) add_hook(&(hooks), NULL, (void_fn)(fn), (ud)) static void add_hook(struct hook **hooks, const char *id, void_fn fn, void *ud) { struct hook *hk = ne_malloc(sizeof (struct hook)), *pos; if (*hooks != NULL) { for (pos = *hooks; pos->next != NULL; pos = pos->next) /* nullop */; pos->next = hk; } else { *hooks = hk; } hk->id = id; hk->fn = fn; hk->userdata = ud; hk->next = NULL; } void ne_hook_create_request(ne_session *sess, ne_create_request_fn fn, void *userdata) { ADD_HOOK(sess->create_req_hooks, fn, userdata); } void ne_hook_pre_send(ne_session *sess, ne_pre_send_fn fn, void *userdata) { ADD_HOOK(sess->pre_send_hooks, fn, userdata); } void ne_hook_post_send(ne_session *sess, ne_post_send_fn fn, void *userdata) { ADD_HOOK(sess->post_send_hooks, fn, userdata); } void ne_hook_destroy_request(ne_session *sess, ne_destroy_req_fn fn, void *userdata) { ADD_HOOK(sess->destroy_req_hooks, fn, userdata); } void ne_hook_destroy_session(ne_session *sess, ne_destroy_sess_fn fn, void *userdata) { ADD_HOOK(sess->destroy_sess_hooks, fn, userdata); } void ne_set_session_private(ne_session *sess, const char *id, void *userdata) { add_hook(&sess->private, id, NULL, userdata); } void ne_set_request_private(ne_request *req, const char *id, void *userdata) { add_hook(&req->private, id, NULL, userdata); } static ssize_t body_string_send(void *userdata, char *buffer, size_t count) { ne_request *req = userdata; if (count == 0) { req->body.buf.left = req->body_size; req->body.buf.pnt = req->body.buf.buffer; } else { /* if body_left == 0 we fall through and return 0. */ if (req->body.buf.left < count) count = req->body.buf.left; memcpy(buffer, req->body.buf.pnt, count); req->body.buf.pnt += count; req->body.buf.left -= count; } return count; } static ssize_t body_fd_send(void *userdata, char *buffer, size_t count) { ne_request *req = userdata; if (count) { return read(req->body.fd, buffer, count); } else { /* rewind since we may have to send it again */ return lseek(req->body.fd, SEEK_SET, 0); } } /* Pulls the request body from the source and pushes it to the given * callback. Returns 0 on success, or NE_* code */ int ne_pull_request_body(ne_request *req, ne_push_fn fn, void *ud) { int ret = 0; char buffer[BUFSIZ]; ssize_t bytes; /* tell the source to start again from the beginning. */ (void) req->body_cb(req->body_ud, NULL, 0); /* TODO: should this attempt to pull exactly the number of bytes * they specified were in the body? Currently it just pulls until * they return zero. That makes it possible to extend this to do * chunked request bodies (i.e. indefinitely long, no C-L), so * this is probably a better long-term interface. */ while ((bytes = req->body_cb(req->body_ud, buffer, sizeof buffer)) > 0) { ret = fn(ud, buffer, bytes); if (ret < 0) break; NE_DEBUG(NE_DBG_HTTPBODY, "Body block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n", bytes, (int)bytes, buffer); } if (bytes < 0) { ne_set_error(req->session, _("Error reading request body.")); ret = NE_ERROR; } return ret; } static int send_with_progress(void *userdata, const char *data, size_t n) { ne_request *req = userdata; int ret; ret = ne_sock_fullwrite(req->session->socket, data, n); if (ret == 0) { req->body_progress += n; req->session->progress_cb(req->session->progress_ud, req->body_progress, req->body_size); } return ret; } /* Sends the request body down the socket. * Returns 0 on success, or NE_* code */ static int send_request_body(ne_request *req) { int ret; NE_DEBUG(NE_DBG_HTTP, "Sending request body...\n"); if (req->session->progress_cb) { /* with progress callbacks. */ req->body_progress = 0; ret = ne_pull_request_body(req, send_with_progress, req); } else { /* without progress callbacks. */ ret = ne_pull_request_body(req, (ne_push_fn)ne_sock_fullwrite, req->session->socket); } NE_DEBUG(NE_DBG_HTTP, "Request body sent: %s.\n", ret?"failed":"okay"); return ret; } /* Lob the User-Agent, connection and host headers in to the request * headers */ static void add_fixed_headers(ne_request *req) { if (req->session->user_agent) { ne_buffer_zappend(req->headers, req->session->user_agent); } /* Send Connection: Keep-Alive to pre-1.1 origin servers to try * harder to get a persistent connection, except if using a proxy * as per 2068 sec 19.7.1. Always add TE: trailers since those * are understood. */ if (!req->session->is_http11 && !req->session->use_proxy) { ne_buffer_zappend(req->headers, "Keep-Alive: " EOL "Connection: TE, Keep-Alive" EOL "TE: trailers" EOL); } else { ne_buffer_zappend(req->headers, "Connection: TE" EOL "TE: trailers" EOL); } } int ne_accept_always(void *userdata, ne_request *req, const ne_status *st) { return 1; } int ne_accept_2xx(void *userdata, ne_request *req, const ne_status *st) { return (st->klass == 2); } /* Handler for the "Transfer-Encoding" response header: treat *any* * such header as implying a chunked response, per the "Protocol * Compliance" statement in the manual. */ static void te_hdr_handler(void *userdata, const char *value) { struct ne_response *resp = userdata; resp->mode = R_CHUNKED; } /* Handler for the "Connection" response header */ static void connection_hdr_handler(void *userdata, const char *value) { ne_request *req = userdata; if (strcasecmp(value, "close") == 0) { req->can_persist = 0; } else if (strcasecmp(value, "Keep-Alive") == 0) { req->can_persist = 1; } } static void clength_hdr_handler(void *userdata, const char *value) { struct ne_response *resp = userdata; size_t len = strtoul(value, NULL, 10); if (len != ULONG_MAX && resp->mode == R_TILLEOF) { resp->mode = R_CLENGTH; resp->length = len; } } ne_request *ne_request_create(ne_session *sess, const char *method, const char *path) { ne_request *req = ne_calloc(sizeof *req); NE_DEBUG(NE_DBG_HTTP, "Creating request...\n"); req->session = sess; req->headers = ne_buffer_create(); /* Add in the fixed headers */ add_fixed_headers(req); /* Set the standard stuff */ req->method = ne_strdup(method); req->method_is_head = (strcmp(method, "HEAD") == 0); /* Add in handlers for all the standard HTTP headers. */ ne_add_response_header_handler(req, "Content-Length", clength_hdr_handler, &req->resp); ne_add_response_header_handler(req, "Transfer-Encoding", te_hdr_handler, &req->resp); ne_add_response_header_handler(req, "Connection", connection_hdr_handler, req); /* Only use an absoluteURI here when absolutely necessary: some * servers can't parse them. */ if (req->session->use_proxy && !req->session->use_ssl && path[0] == '/') req->uri = ne_concat(req->session->scheme, "://", req->session->server.hostport, path, NULL); else req->uri = ne_strdup(path); { struct hook *hk; NE_DEBUG(NE_DBG_HTTP, "Running request create hooks.\n"); for (hk = sess->create_req_hooks; hk != NULL; hk = hk->next) { ne_create_request_fn fn = (ne_create_request_fn)hk->fn; fn(req, hk->userdata, method, req->uri); } } NE_DEBUG(NE_DBG_HTTP, "Request created.\n"); return req; } static void set_body_size(ne_request *req, size_t size) { req->body_size = size; ne_print_request_header(req, "Content-Length", "%" NE_FMT_SIZE_T, size); } void ne_set_request_body_buffer(ne_request *req, const char *buffer, size_t size) { req->body.buf.buffer = buffer; req->body_cb = body_string_send; req->body_ud = req; set_body_size(req, size); } void ne_set_request_body_provider(ne_request *req, size_t bodysize, ne_provide_body provider, void *ud) { req->body_cb = provider; req->body_ud = ud; set_body_size(req, bodysize); } int ne_set_request_body_fd(ne_request *req, int fd) { struct stat bodyst; /* Get file length */ if (fstat(fd, &bodyst) < 0) { char err[200]; ne_strerror(errno, err, sizeof err); ne_set_error(req->session, _("Could not determine file length: %s"), err); NE_DEBUG(NE_DBG_HTTP, "Stat failed: %s\n", err); return -1; } req->body.fd = fd; req->body_cb = body_fd_send; req->body_ud = req; set_body_size(req, bodyst.st_size); return 0; } void ne_add_request_header(ne_request *req, const char *name, const char *value) { ne_buffer_concat(req->headers, name, ": ", value, EOL, NULL); } void ne_print_request_header(ne_request *req, const char *name, const char *format, ...) { va_list params; char buf[BUFSIZ]; va_start(params, format); ne_vsnprintf(buf, sizeof buf, format, params); va_end(params); ne_buffer_concat(req->headers, name, ": ", buf, EOL, NULL); } void ne_add_response_header_handler(ne_request *req, const char *name, ne_header_handler hdl, void *userdata) { struct header_handler *new = ne_calloc(sizeof *new); unsigned int hash; new->name = ne_strdup(name); new->handler = hdl; new->userdata = userdata; hash = hash_and_lower(new->name); new->next = req->header_handlers[hash]; req->header_handlers[hash] = new; } void ne_add_response_header_catcher(ne_request *req, ne_header_handler hdl, void *userdata) { struct header_handler *new = ne_calloc(sizeof *new); new->handler = hdl; new->userdata = userdata; new->next = req->header_catchers; req->header_catchers = new; } void ne_add_response_body_reader(ne_request *req, ne_accept_response acpt, ne_block_reader rdr, void *userdata) { struct body_reader *new = ne_malloc(sizeof *new); new->accept_response = acpt; new->handler = rdr; new->userdata = userdata; new->next = req->body_readers; req->body_readers = new; } void ne_request_destroy(ne_request *req) { struct body_reader *rdr, *next_rdr; struct header_handler *hdlr, *next_hdlr; struct hook *hk, *next_hk; int n; ne_free(req->uri); ne_free(req->method); for (rdr = req->body_readers; rdr != NULL; rdr = next_rdr) { next_rdr = rdr->next; ne_free(rdr); } for (hdlr = req->header_catchers; hdlr != NULL; hdlr = next_hdlr) { next_hdlr = hdlr->next; ne_free(hdlr); } for (n = 0; n < HH_HASHSIZE; n++) { for (hdlr = req->header_handlers[n]; hdlr != NULL; hdlr = next_hdlr) { next_hdlr = hdlr->next; ne_free(hdlr->name); ne_free(hdlr); } } ne_buffer_destroy(req->headers); NE_DEBUG(NE_DBG_HTTP, "Running destroy hooks.\n"); for (hk = req->session->destroy_req_hooks; hk; hk = hk->next) { ne_destroy_req_fn fn = (ne_destroy_req_fn)hk->fn; fn(req, hk->userdata); } for (hk = req->private; hk; hk = next_hk) { next_hk = hk->next; ne_free(hk); } if (req->status.reason_phrase) ne_free(req->status.reason_phrase); NE_DEBUG(NE_DBG_HTTP, "Request ends.\n"); ne_free(req); } /* Reads a block of the response into buffer, which is of size buflen. * Returns number of bytes read, 0 on end-of-response, or NE_* on error. * TODO?: only make one actual read() call in here... */ static int read_response_block(ne_request *req, struct ne_response *resp, char *buffer, size_t *buflen) { size_t willread; ssize_t readlen; ne_socket *sock = req->session->socket; switch (resp->mode) { case R_CHUNKED: /* We are doing a chunked transfer-encoding. * It goes: `SIZE CRLF CHUNK CRLF SIZE CRLF CHUNK CRLF ...' * ended by a `CHUNK CRLF 0 CRLF', a 0-sized chunk. * The slight complication is that we have to cope with * partial reads of chunks. * For this reason, resp.chunk_left contains the number of * bytes left to read in the current chunk. */ if (resp->chunk_left == 0) { unsigned long int chunk_len; char *ptr; /* We are at the start of a new chunk. */ NE_DEBUG(NE_DBG_HTTP, "New chunk.\n"); SOCK_ERR(req, ne_sock_readline(sock, buffer, *buflen), _("Could not read chunk size")); NE_DEBUG(NE_DBG_HTTP, "[Chunk Size] < %s", buffer); chunk_len = strtoul(buffer, &ptr, 16); /* limit chunk size to <= UINT_MAX, so it will probably * fit in a size_t. */ if (ptr == buffer || chunk_len == ULONG_MAX || chunk_len > UINT_MAX) { return aborted(req, _("Could not parse chunk size"), 0); } NE_DEBUG(NE_DBG_HTTP, "Got chunk size: %lu\n", chunk_len); if (chunk_len == 0) { /* Zero-size chunk == end of response. */ NE_DEBUG(NE_DBG_HTTP, "Zero-size chunk.\n"); *buflen = 0; return NE_OK; } resp->chunk_left = chunk_len; } willread = resp->chunk_left; break; case R_CLENGTH: willread = resp->left; break; case R_TILLEOF: willread = *buflen; break; case R_NO_BODY: default: willread = 0; break; } if (willread > *buflen) willread = *buflen; else if (willread == 0) { *buflen = 0; return 0; } NE_DEBUG(NE_DBG_HTTP, "Reading %" NE_FMT_SIZE_T " bytes of response body.\n", willread); readlen = ne_sock_read(sock, buffer, willread); /* EOF is only valid when response body is delimited by it. * Strictly, an SSL truncation should not be treated as an EOF in * any case, but SSL servers are just too buggy. */ if (resp->mode == R_TILLEOF && (readlen == NE_SOCK_CLOSED || readlen == NE_SOCK_TRUNC)) { NE_DEBUG(NE_DBG_HTTP, "Got EOF.\n"); req->can_persist = 0; readlen = 0; } else if (readlen < 0) { return aborted(req, _("Could not read response body"), readlen); } else { NE_DEBUG(NE_DBG_HTTP, "Got %" NE_FMT_SSIZE_T " bytes.\n", readlen); } /* safe to cast: readlen guaranteed to be >= 0 above */ *buflen = (size_t)readlen; NE_DEBUG(NE_DBG_HTTPBODY, "Read block (%" NE_FMT_SSIZE_T " bytes):\n[%.*s]\n", readlen, (int)readlen, buffer); if (resp->mode == R_CHUNKED) { resp->chunk_left -= readlen; if (resp->chunk_left == 0) { char crlfbuf[2]; /* If we've read a whole chunk, read a CRLF */ readlen = ne_sock_fullread(sock, crlfbuf, 2); if (readlen < 0) return aborted(req, _("Could not read chunk delimiter"), readlen); else if (crlfbuf[0] != '\r' || crlfbuf[1] != '\n') return aborted(req, _("Chunk delimiter was invalid"), 0); } } else if (resp->mode == R_CLENGTH) { resp->left -= readlen; } return NE_OK; } ssize_t ne_read_response_block(ne_request *req, char *buffer, size_t buflen) { struct body_reader *rdr; size_t readlen = buflen; if (read_response_block(req, &req->resp, buffer, &readlen)) return -1; req->resp.total += readlen; if (req->session->progress_cb) { req->session->progress_cb(req->session->progress_ud, req->resp.total, (req->resp.mode==R_CLENGTH)?req->resp.length:-1); } /* TODO: call the readers when this fails too. */ for (rdr = req->body_readers; rdr!=NULL; rdr=rdr->next) { if (rdr->use) rdr->handler(rdr->userdata, buffer, readlen); } return readlen; } /* Build the request string, returning the buffer. */ static ne_buffer *build_request(ne_request *req) { struct hook *hk; ne_buffer *buf = ne_buffer_create(); /* Add Request-Line and Host header: */ ne_buffer_concat(buf, req->method, " ", req->uri, " HTTP/1.1" EOL, "Host: ", req->session->server.hostport, EOL, NULL); /* Add custom headers: */ ne_buffer_append(buf, req->headers->data, ne_buffer_size(req->headers)); #define E100 "Expect: 100-continue" EOL if (req->use_expect100) ne_buffer_append(buf, E100, strlen(E100)); NE_DEBUG(NE_DBG_HTTP, "Running pre_send hooks\n"); for (hk = req->session->pre_send_hooks; hk!=NULL; hk = hk->next) { ne_pre_send_fn fn = (ne_pre_send_fn)hk->fn; fn(req, hk->userdata, buf); } ne_buffer_append(buf, "\r\n", 2); return buf; } #ifdef NE_DEBUGGING #define DEBUG_DUMP_REQUEST(x) dump_request(x) static void dump_request(const char *request) { if ((NE_DBG_HTTPPLAIN&ne_debug_mask) == NE_DBG_HTTPPLAIN) { /* Display everything mode */ NE_DEBUG(NE_DBG_HTTP, "Sending request headers:\n%s", request); } else { /* Blank out the Authorization paramaters */ char *reqdebug = ne_strdup(request), *pnt = reqdebug; while ((pnt = strstr(pnt, "Authorization: ")) != NULL) { for (pnt += 15; *pnt != '\r' && *pnt != '\0'; pnt++) { *pnt = 'x'; } } NE_DEBUG(NE_DBG_HTTP, "Sending request headers:\n%s", reqdebug); ne_free(reqdebug); } } #else #define DEBUG_DUMP_REQUEST(x) #endif /* DEBUGGING */ /* remove trailing EOL from 'buf', where strlen(buf) == *len. *len is * adjusted in accordance with any changes made to the string to * remain equal to strlen(buf). */ static inline void strip_eol(char *buf, ssize_t *len) { char *pnt = &buf[*len-1]; while (pnt >= buf && (*pnt == '\r' || *pnt == '\n')) { *pnt-- = '\0'; (*len)--; } } /* For accurate persistent connection handling, for any write() or * read() operation for a new request on an already-open connection, * an EOF or RST error MUST be treated as a persistent connection * timeout, and the request retried on a new connection. Once a * read() operation has succeeded, any subsequent error MUST be * treated as fatal. A 'retry' flag is used; retry=1 represents the * first case, retry=0 the latter. */ /* RETRY_RET() crafts a function return value given the 'retry' flag, * the socket error 'code', and the return value 'acode' from the * aborted() function. */ #define RETRY_RET(retry, code, acode) \ ((((code) == NE_SOCK_CLOSED || (code) == NE_SOCK_RESET || \ (code) == NE_SOCK_TRUNC) && retry) ? NE_RETRY : (acode)) /* Read and parse response status-line into 'status'. 'retry' is non-zero * if an NE_RETRY should be returned if an EOF is received. */ static int read_status_line(ne_request *req, ne_status *status, int retry) { char *buffer = req->respbuf; ssize_t ret; ret = ne_sock_readline(req->session->socket, buffer, sizeof req->respbuf); if (ret <= 0) { int aret = aborted(req, _("Could not read status line"), ret); return RETRY_RET(retry, ret, aret); } NE_DEBUG(NE_DBG_HTTP, "[status-line] < %s", buffer); strip_eol(buffer, &ret); if (status->reason_phrase) ne_free(status->reason_phrase); memset(status, 0, sizeof *status); if (ne_parse_statusline(buffer, status)) return aborted(req, _("Could not parse response status line."), 0); return 0; } /* Discard a set of message headers. */ static int discard_headers(ne_request *req) { do { SOCK_ERR(req, ne_sock_readline(req->session->socket, req->respbuf, sizeof req->respbuf), _("Could not read interim response headers")); NE_DEBUG(NE_DBG_HTTP, "[discard] < %s", req->respbuf); } while (strcmp(req->respbuf, EOL) != 0); return NE_OK; } /* Send the request, and read the response Status-Line. Returns: * NE_RETRY connection closed by server; persistent connection * timeout * NE_OK success * NE_* error * On NE_RETRY and NE_* responses, the connection will have been * closed already. */ static int send_request(ne_request *req, const ne_buffer *request) { ne_session *sess = req->session; ssize_t ret = NE_OK; int sentbody = 0; /* zero until body has been sent. */ int retry; /* non-zero whilst the request should be retried */ ne_status *status = &req->status; /* Send the Request-Line and headers */ NE_DEBUG(NE_DBG_HTTP, "Sending request-line and headers:\n"); /* Open the connection if necessary */ HTTP_ERR(open_connection(req)); /* Allow retry if a persistent connection has been used. */ retry = sess->persisted; ret = ne_sock_fullwrite(req->session->socket, request->data, ne_buffer_size(request)); if (ret < 0) { int aret = aborted(req, _("Could not send request"), ret); return RETRY_RET(retry, ret, aret); } if (!req->use_expect100 && req->body_size > 0) { /* Send request body, if not using 100-continue. */ ret = send_request_body(req); if (ret < 0) { int aret = aborted(req, _("Could not send request body"), ret); return RETRY_RET(sess, ret, aret); } } NE_DEBUG(NE_DBG_HTTP, "Request sent; retry is %d.\n", retry); /* Loop eating interim 1xx responses (RFC2616 says these MAY be * sent by the server, even if 100-continue is not used). */ while ((ret = read_status_line(req, status, retry)) == NE_OK && status->klass == 1) { NE_DEBUG(NE_DBG_HTTP, "Interim %d response.\n", status->code); retry = 0; /* successful read() => never retry now. */ /* Discard headers with the interim response. */ if ((ret = discard_headers(req)) != NE_OK) break; if (req->use_expect100 && (status->code == 100) && !sentbody) { /* Send the body after receiving the first 100 Continue */ if ((ret = send_request_body(req)) != NE_OK) break; sentbody = 1; } } return ret; } /* Read a message header from sock into buf, which has size 'buflen'. * * Returns: * NE_RETRY: Success, read a header into buf. * NE_OK: End of headers reached. * NE_ERROR: Error (session error is set). */ static int read_message_header(ne_request *req, char *buf, size_t buflen) { ssize_t n; ne_socket *sock = req->session->socket; n = ne_sock_readline(sock, buf, buflen); if (n <= 0) return aborted(req, _("Error reading response headers"), n); NE_DEBUG(NE_DBG_HTTP, "[hdr] %s", buf); strip_eol(buf, &n); if (n == 0) { NE_DEBUG(NE_DBG_HTTP, "End of headers.\n"); return NE_OK; } buf += n; buflen -= n; while (buflen > 0) { char ch; /* Collect any extra lines into buffer */ SOCK_ERR(req, ne_sock_peek(sock, &ch, 1), _("Error reading response headers")); if (ch != ' ' && ch != '\t') { /* No continuation of this header: stop reading. */ return NE_RETRY; } /* Otherwise, read the next line onto the end of 'buf'. */ n = ne_sock_readline(sock, buf, buflen); if (n <= 0) { return aborted(req, _("Error reading response headers"), n); } NE_DEBUG(NE_DBG_HTTP, "[cont] %s", buf); strip_eol(buf, &n); /* assert(buf[0] == ch), which implies len(buf) > 0. * Otherwise the TCP stack is lying, but we'll be paranoid. * This might be a \t, so replace it with a space to be * friendly to applications (2616 says we MAY do this). */ if (n) buf[0] = ' '; /* ready for the next header. */ buf += n; buflen -= n; } ne_set_error(req->session, _("Response header too long")); return NE_ERROR; } /* Apache's default is 100, seems reasonable. */ #define MAX_HEADER_FIELDS 100 /* Read response headers. Returns NE_* code, sets session error. */ static int read_response_headers(ne_request *req) { char hdr[8192]; /* max header length */ int ret, count = 0; while ((ret = read_message_header(req, hdr, sizeof hdr)) == NE_RETRY && ++count < MAX_HEADER_FIELDS) { struct header_handler *hdl; char *pnt; unsigned int hash = 0; for (hdl = req->header_catchers; hdl != NULL; hdl = hdl->next) hdl->handler(hdl->userdata, hdr); /* Strip any trailing whitespace */ pnt = hdr + strlen(hdr) - 1; while (pnt > hdr && (*pnt == ' ' || *pnt == '\t')) *pnt-- = '\0'; /* Convert the header name to lower case and hash it. */ for (pnt = hdr; (*pnt != '\0' && *pnt != ':' && *pnt != ' ' && *pnt != '\t'); pnt++) { *pnt = tolower(*pnt); hash = HH_ITERATE(hash,*pnt); } /* Skip over any whitespace before the colon. */ while (*pnt == ' ' || *pnt == '\t') *pnt++ = '\0'; /* ignore header lines which lack a ':'. */ if (*pnt != ':') continue; /* NUL-terminate at the colon (when no whitespace before) */ *pnt++ = '\0'; /* Skip any whitespace after the colon... */ while (*pnt == ' ' || *pnt == '\t') pnt++; /* pnt now points to the header value. */ NE_DEBUG(NE_DBG_HTTP, "Header Name: [%s], Value: [%s]\n", hdr, pnt); /* Iterate through the header handlers */ for (hdl = req->header_handlers[hash]; hdl != NULL; hdl = hdl->next) { if (strcmp(hdr, hdl->name) == 0) hdl->handler(hdl->userdata, pnt); } } if (count == MAX_HEADER_FIELDS) ret = aborted( req, _("Response exceeded maximum number of header fields."), 0); return ret; } static int lookup_host(ne_session *sess, struct host_info *info) { NE_DEBUG(NE_DBG_HTTP, "Doing DNS lookup on %s...\n", info->hostname); if (sess->notify_cb) sess->notify_cb(sess->notify_ud, ne_conn_namelookup, info->hostname); info->address = ne_addr_resolve(info->hostname, 0); if (ne_addr_result(info->address)) { char buf[256]; ne_set_error(sess, _("Could not resolve hostname `%s': %s"), info->hostname, ne_addr_error(info->address, buf, sizeof buf)); ne_addr_destroy(info->address); info->address = NULL; return NE_LOOKUP; } else { return NE_OK; } } int ne_begin_request(ne_request *req) { struct body_reader *rdr; struct host_info *host; ne_buffer *data; const ne_status *const st = &req->status; int ret; /* Resolve hostname if necessary. */ host = req->session->use_proxy?&req->session->proxy:&req->session->server; if (host->address == NULL) HTTP_ERR(lookup_host(req->session, host)); req->resp.mode = R_TILLEOF; /* FIXME: Determine whether to use the Expect: 100-continue header. */ req->use_expect100 = (req->session->expect100_works > -1) && (req->body_size > HTTP_EXPECT_MINSIZE) && req->session->is_http11; /* Build the request string, and send it */ data = build_request(req); DEBUG_DUMP_REQUEST(data->data); ret = send_request(req, data); /* Retry this once after a persistent connection timeout. */ if (ret == NE_RETRY && !req->session->no_persist) { NE_DEBUG(NE_DBG_HTTP, "Persistent connection timed out, retrying.\n"); ret = send_request(req, data); } ne_buffer_destroy(data); if (ret != NE_OK) return ret; /* Determine whether server claims HTTP/1.1 compliance. */ req->session->is_http11 = (st->major_version == 1 && st->minor_version > 0) || st->major_version > 1; /* Persistent connections supported implicitly in HTTP/1.1 */ if (req->session->is_http11) req->can_persist = 1; ne_set_error(req->session, "%d %s", st->code, st->reason_phrase); /* Read the headers */ HTTP_ERR(read_response_headers(req)); #ifdef NEON_SSL /* Special case for CONNECT handling: the response has no body, * and the connection can persist. */ if (req->session->in_connect && st->klass == 2) { req->resp.mode = R_NO_BODY; req->can_persist = 1; } #endif /* HEAD requests and 204, 304 responses have no response body, * regardless of what headers are present. */ if (req->method_is_head || st->code==204 || st->code==304) req->resp.mode = R_NO_BODY; /* Prepare for reading the response entity-body. Call each of the * body readers and ask them whether they want to accept this * response or not. */ for (rdr = req->body_readers; rdr != NULL; rdr=rdr->next) { rdr->use = rdr->accept_response(rdr->userdata, req, st); } req->resp.left = req->resp.length; req->resp.chunk_left = 0; return NE_OK; } int ne_end_request(ne_request *req) { struct hook *hk; int ret = NE_OK; /* Read headers in chunked trailers */ if (req->resp.mode == R_CHUNKED) HTTP_ERR(read_response_headers(req)); NE_DEBUG(NE_DBG_HTTP, "Running post_send hooks\n"); for (hk = req->session->post_send_hooks; ret == NE_OK && hk != NULL; hk = hk->next) { ne_post_send_fn fn = (ne_post_send_fn)hk->fn; ret = fn(req, hk->userdata, &req->status); } /* Close the connection if persistent connections are disabled or * not supported by the server. */ if (req->session->no_persist || !req->can_persist) ne_close_connection(req->session); else req->session->persisted = 1; return ret; } int ne_request_dispatch(ne_request *req) { int ret; /* Loop sending the request: * Retry whilst authentication fails and we supply it. */ do { ssize_t len; HTTP_ERR(ne_begin_request(req)); do { len = ne_read_response_block(req, req->respbuf, sizeof req->respbuf); } while (len > 0); if (len < 0) { return NE_ERROR; } ret = ne_end_request(req); } while (ret == NE_RETRY); NE_DEBUG(NE_DBG_HTTP | NE_DBG_FLUSH, "Request ends, status %d class %dxx, error line:\n%s\n", req->status.code, req->status.klass, req->session->error); return ret; } const ne_status *ne_get_status(const ne_request *req) { return &req->status; } ne_session *ne_get_session(const ne_request *req) { return req->session; } #ifdef NEON_SSL /* Create a CONNECT tunnel through the proxy server. * Returns HTTP_* */ static int proxy_tunnel(ne_session *sess) { /* Hack up an HTTP CONNECT request... */ ne_request *req; int ret = NE_OK; char ruri[200]; /* Can't use server.hostport here; Request-URI must include `:port' */ ne_snprintf(ruri, sizeof ruri, "%s:%u", sess->server.hostname, sess->server.port); req = ne_request_create(sess, "CONNECT", ruri); sess->in_connect = 1; ret = ne_request_dispatch(req); sess->in_connect = 0; sess->persisted = 0; /* don't treat this is a persistent connection. */ if (ret != NE_OK || !sess->connected || req->status.klass != 2) { ne_set_error (sess, _("Could not create SSL connection through proxy server")); ret = NE_ERROR; } ne_request_destroy(req); return ret; } #endif /* Make new TCP connection to server at 'host' of type 'name'. Note * that once a connection to a particular network address has * succeeded, that address will be used first for the next attempt to * connect. */ /* TODO: an alternate implementation could always cycle through the * addresses: this could ease server load, but could hurt SSL session * caching for SSL sessions, which would increase server load. */ static int do_connect(ne_request *req, struct host_info *host, const char *err) { ne_session *const sess = req->session; int ret; if ((sess->socket = ne_sock_create()) == NULL) { ne_set_error(sess, _("Could not create socket")); return NE_ERROR; } if (host->current == NULL) host->current = ne_addr_first(host->address); do { notify_status(sess, ne_conn_connecting, host->hostport); #ifdef NE_DEBUGGING if (ne_debug_mask & NE_DBG_HTTP) { char buf[150]; NE_DEBUG(NE_DBG_HTTP, "Connecting to %s\n", ne_iaddr_print(host->current, buf, sizeof buf)); } #endif ret = ne_sock_connect(sess->socket, host->current, host->port); } while (ret && /* try the next address... */ (host->current = ne_addr_next(host->address)) != NULL); if (ret) { ne_set_error(sess, "%s: %s", err, ne_sock_error(sess->socket)); ne_sock_close(sess->socket); return NE_CONNECT; } notify_status(sess, ne_conn_connected, sess->proxy.hostport); if (sess->rdtimeout) ne_sock_read_timeout(sess->socket, sess->rdtimeout); sess->connected = 1; /* clear persistent connection flag. */ sess->persisted = 0; return NE_OK; } static int open_connection(ne_request *req) { ne_session *sess = req->session; int ret; if (sess->connected) return NE_OK; if (!sess->use_proxy) ret = do_connect(req, &sess->server, _("Could not connect to server")); else ret = do_connect(req, &sess->proxy, _("Could not connect to proxy server")); if (ret != NE_OK) return ret; #ifdef NEON_SSL /* Negotiate SSL layer if required. */ if (sess->use_ssl && !sess->in_connect) { /* CONNECT tunnel */ if (req->session->use_proxy) ret = proxy_tunnel(sess); if (ret == NE_OK) ret = ne_negotiate_ssl(req); /* This is probably only really needed for ne_negotiate_ssl * failures as proxy_tunnel will fail via aborted(). */ if (ret != NE_OK) ne_close_connection(sess); } #endif return ret; }