Compliance: remove Content-Length header if Transfer-Encoding is present. If after HTTP header parsing we have both "Transfer-Encoding: chunked" and Content-Length headers, remove the Content-Length entry. The adjusted behavior follows httpbis recommendations (ticket #95, part 2). The old client-side code forwarded the original Content-Length header which did not match the [dechunked] response, resulting in a malformed response. HttpHeader::chunked() method added to check if HTTP headers contain chunked Transfer-Encoding header. Use this method in code that checks for chunked encoding. Co-Advisor test cases: test_case/rfc2616/chunked-1p0-badClen-toClt test_case/rfc2616/chunked-1p1-badClen-toClt === modified file 'src/HttpHeader.cc' --- src/HttpHeader.cc 2010-05-31 19:51:06 +0000 +++ src/HttpHeader.cc 2010-08-18 16:29:10 +0000 @@ -630,40 +630,45 @@ HttpHeader::parse(const char *header_sta delete e; goto reset; } } } if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) { debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2, "WARNING: found whitespace in HTTP header name {" << getStringPrefix(field_start, field_end) << "}"); if (!Config.onoff.relaxed_header_parser) { delete e; goto reset; } } addEntry(e); } + if (chunked()) { + // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding + delById(HDR_CONTENT_LENGTH); + } + PROF_stop(HttpHeaderParse); return 1; /* even if no fields where found, it is a valid header */ reset: PROF_stop(HttpHeaderParse); return reset(); } /* packs all the entries using supplied packer */ void HttpHeader::packInto(Packer * p) const { HttpHeaderPos pos = HttpHeaderInitPos; const HttpHeaderEntry *e; assert(p); debugs(55, 7, "packing hdr: (" << this << ")"); /* pack all entries one by one */ while ((e = getEntry(&pos))) e->packInto(p); /* Pack in the "special" entries */ === modified file 'src/HttpHeader.h' --- src/HttpHeader.h 2010-03-05 07:10:40 +0000 +++ src/HttpHeader.h 2010-08-18 20:44:09 +0000 @@ -238,48 +238,56 @@ public: void putCc(const HttpHdrCc * cc); void putContRange(const HttpHdrContRange * cr); void putRange(const HttpHdrRange * range); void putSc(HttpHdrSc *sc); void putExt(const char *name, const char *value); int getInt(http_hdr_type id) const; int64_t getInt64(http_hdr_type id) const; time_t getTime(http_hdr_type id) const; const char *getStr(http_hdr_type id) const; const char *getLastStr(http_hdr_type id) const; HttpHdrCc *getCc() const; HttpHdrRange *getRange() const; HttpHdrSc *getSc() const; HttpHdrContRange *getContRange() const; const char *getAuth(http_hdr_type id, const char *auth_scheme) const; ETag getETag(http_hdr_type id) const; TimeOrTag getTimeOrTag(http_hdr_type id) const; int hasListMember(http_hdr_type id, const char *member, const char separator) const; int hasByNameListMember(const char *name, const char *member, const char separator) const; void removeHopByHopEntries(); + inline bool chunked() const; ///< whether message uses chunked Transfer-Encoding /* protected, do not use these, use interface functions instead */ Vector entries; /**< parsed fields in raw format */ HttpHeaderMask mask; /**< bit set <=> entry present */ http_hdr_owner_type owner; /**< request or reply */ int len; /**< length when packed, not counting terminating null-byte */ protected: /** \deprecated Public access replaced by removeHopByHopEntries() */ void removeConnectionHeaderEntries(); private: HttpHeaderEntry *findLastEntry(http_hdr_type id) const; /// Made it non-copyable. Our destructor is a bit nasty... HttpHeader(const HttpHeader &); //assignment is used by the reset method, can't block it.. //const HttpHeader operator=(const HttpHeader &); }; +inline bool +HttpHeader::chunked() const +{ + return has(HDR_TRANSFER_ENCODING) && + hasListMember(HDR_TRANSFER_ENCODING, "chunked", ','); +} + extern int httpHeaderParseQuotedString (const char *start, String *val); SQUIDCEXTERN int httpHeaderHasByNameListMember(const HttpHeader * hdr, const char *name, const char *member, const char separator); SQUIDCEXTERN void httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask); int httpMsgIsPersistent(HttpVersion const &http_ver, const HttpHeader * hdr); SQUIDCEXTERN void httpHeaderCalcMask(HttpHeaderMask * mask, http_hdr_type http_hdr_type_enums[], size_t count); #endif /* SQUID_HTTPHEADER_H */ === modified file 'src/HttpReply.cc' --- src/HttpReply.cc 2010-07-13 16:43:00 +0000 +++ src/HttpReply.cc 2010-08-18 01:02:52 +0000 @@ -523,41 +523,41 @@ HttpReply::httpMsgParseError() * Indicate whether or not we would usually expect an entity-body * along with this response */ bool HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const { bool expectBody = true; if (req_method == METHOD_HEAD) expectBody = false; else if (sline.status == HTTP_NO_CONTENT) expectBody = false; else if (sline.status == HTTP_NOT_MODIFIED) expectBody = false; else if (sline.status < HTTP_OK) expectBody = false; else expectBody = true; if (expectBody) { - if (header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) + if (header.chunked()) theSize = -1; else if (content_length >= 0) theSize = content_length; else theSize = -1; } return expectBody; } bool HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize) { calcMaxBodySize(request); debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax); return bodySizeMax >= 0 && receivedSize > bodySizeMax; } bool HttpReply::expectedBodyTooLarge(HttpRequest& request) === modified file 'src/HttpRequest.cc' --- src/HttpRequest.cc 2010-07-13 16:43:00 +0000 +++ src/HttpRequest.cc 2010-08-18 01:02:23 +0000 @@ -481,49 +481,49 @@ void HttpRequest::packFirstLineInto(Pack } /* * Indicate whether or not we would usually expect an entity-body * along with this request */ bool HttpRequest::expectingBody(const HttpRequestMethod& unused, int64_t& theSize) const { bool expectBody = false; /* * GET and HEAD don't usually have bodies, but we should be prepared * to accept one if the request_entities directive is set */ if (method == METHOD_GET || method == METHOD_HEAD) expectBody = Config.onoff.request_entities ? true : false; else if (method == METHOD_PUT || method == METHOD_POST) expectBody = true; - else if (header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) + else if (header.chunked()) expectBody = true; else if (content_length >= 0) expectBody = true; else expectBody = false; if (expectBody) { - if (header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) + if (header.chunked()) theSize = -1; else if (content_length >= 0) theSize = content_length; else theSize = -1; } return expectBody; } /* * Create a Request from a URL and METHOD. * * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL. * If the request cannot be created cleanly, NULL is returned */ HttpRequest * HttpRequest::CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method) { return urlParse(method, url, NULL); === modified file 'src/client_side.cc' --- src/client_side.cc 2010-08-07 14:22:54 +0000 +++ src/client_side.cc 2010-08-18 01:07:10 +0000 @@ -1950,42 +1950,41 @@ prepareTransparentURL(ConnStateData * co } else { /* Put the local socket IP address as the hostname. */ int url_sz = strlen(url) + 32 + Config.appendDomainLen; http->uri = (char *)xcalloc(url_sz, 1); snprintf(http->uri, url_sz, "%s://%s:%d%s", http->getConn()->port->protocol, http->getConn()->me.NtoA(ntoabuf,MAX_IPSTRLEN), http->getConn()->me.GetPort(), url); debugs(33, 5, "TRANSPARENT REWRITE: '" << http->uri << "'"); } } // Temporary hack helper: determine whether the request is chunked, expensive static bool isChunkedRequest(const HttpParser *hp) { HttpRequest request; if (!request.parseHeader(HttpParserHdrBuf(hp), HttpParserHdrSz(hp))) return false; - return request.header.has(HDR_TRANSFER_ENCODING) && - request.header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ','); + return request.header.chunked(); } /** * parseHttpRequest() * * Returns * NULL on incomplete requests * a ClientSocketContext structure on success or failure. * Sets result->flags.parsed_ok to 0 if failed to parse the request. * Sets result->flags.parsed_ok to 1 if we have a good request. */ static ClientSocketContext * parseHttpRequest(ConnStateData *conn, HttpParser *hp, HttpRequestMethod * method_p, HttpVersion *http_ver) { char *req_hdr = NULL; char *end; size_t req_sz; ClientHttpRequest *http; ClientSocketContext *result; === modified file 'src/http.cc' --- src/http.cc 2010-08-13 07:53:08 +0000 +++ src/http.cc 2010-08-18 01:01:41 +0000 @@ -709,41 +709,41 @@ HttpStateData::processReplyHeader() if (newrep->sline.protocol == PROTO_HTTP && newrep->sline.status >= 100 && newrep->sline.status < 200) { #if WHEN_HTTP11_EXPECT_HANDLED /* When HTTP/1.1 check if the client is expecting a 1xx reply and maybe pass it on */ if (orig_request->header.has(HDR_EXPECT)) { // TODO: pass to the client anyway? } #endif delete newrep; debugs(11, 2, HERE << "1xx headers consume " << header_bytes_read << " bytes header."); header_bytes_read = 0; if (reply_bytes_read > 0) debugs(11, 2, HERE << "1xx headers consume " << reply_bytes_read << " bytes reply."); reply_bytes_read = 0; ctx_exit(ctx); processReplyHeader(); return; } flags.chunked = 0; - if (newrep->sline.protocol == PROTO_HTTP && newrep->header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) { + if (newrep->sline.protocol == PROTO_HTTP && newrep->header.chunked()) { flags.chunked = 1; httpChunkDecoder = new ChunkedCodingParser; } if (!peerSupportsConnectionPinning()) orig_request->flags.connection_auth_disabled = 1; HttpReply *vrep = setVirginReply(newrep); flags.headers_parsed = 1; keepaliveAccounting(vrep); checkDateSkew(vrep); processSurrogateControl (vrep); /** \todo IF the reply is a 1.0 reply, AND it has a Connection: Header * Parse the header and remove all referenced headers */