Hello community, here is the log from the commit of package libhtp for openSUSE:Factory checked in at 2019-12-17 16:55:02 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/libhtp (Old) and /work/SRC/openSUSE:Factory/.libhtp.new.4691 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libhtp" Tue Dec 17 16:55:02 2019 rev:3 rq:757560 version:0.5.32 Changes: -------- --- /work/SRC/openSUSE:Factory/libhtp/libhtp.changes 2019-10-03 14:08:24.604262272 +0200 +++ /work/SRC/openSUSE:Factory/.libhtp.new.4691/libhtp.changes 2019-12-17 16:55:03.581350693 +0100 @@ -1,0 +2,6 @@ +Sun Dec 15 10:23:41 UTC 2019 - Martin Hauke <[email protected]> + +- Update to version 0.5.32 + * bug fixes around pipelining + +------------------------------------------------------------------- Old: ---- libhtp-0.5.31.tar.gz New: ---- libhtp-0.5.32.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ libhtp.spec ++++++ --- /var/tmp/diff_new_pack.seeazf/_old 2019-12-17 16:55:04.125350887 +0100 +++ /var/tmp/diff_new_pack.seeazf/_new 2019-12-17 16:55:04.125350887 +0100 @@ -19,7 +19,7 @@ %define sover 2 %define lname %{name}%{sover} Name: libhtp -Version: 0.5.31 +Version: 0.5.32 Release: 0 Summary: HTTP normalizer and parser License: BSD-3-Clause ++++++ libhtp-0.5.31.tar.gz -> libhtp-0.5.32.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/ChangeLog new/libhtp-0.5.32/ChangeLog --- old/libhtp-0.5.31/ChangeLog 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/ChangeLog 2019-12-13 10:30:17.000000000 +0100 @@ -1,3 +1,8 @@ +0.5.32 (13 December 2019) +-------------------------- + +- bug fixes around pipelining + 0.5.31 (24 September 2019) -------------------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/VERSION new/libhtp-0.5.32/VERSION --- old/libhtp-0.5.31/VERSION 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/VERSION 2019-12-13 10:30:17.000000000 +0100 @@ -1,2 +1,2 @@ # This file is intended to be sourced by sh -PKG_VERSION=0.5.31 +PKG_VERSION=0.5.32 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/htp/htp_connection_parser.c new/libhtp-0.5.32/htp/htp_connection_parser.c --- old/libhtp-0.5.31/htp/htp_connection_parser.c 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/htp/htp_connection_parser.c 2019-12-13 10:30:17.000000000 +0100 @@ -44,6 +44,18 @@ connp->last_error = NULL; } +void htp_connp_req_close(htp_connp_t *connp, const htp_time_t *timestamp) { + if (connp == NULL) return; + + // Update internal flags + if (connp->in_status != HTP_STREAM_ERROR) + connp->in_status = HTP_STREAM_CLOSED; + + // Call the parsers one last time, which will allow them + // to process the events that depend on stream closure + htp_connp_req_data(connp, timestamp, NULL, 0); +} + void htp_connp_close(htp_connp_t *connp, const htp_time_t *timestamp) { if (connp == NULL) return; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/htp/htp_connection_parser.h new/libhtp-0.5.32/htp/htp_connection_parser.h --- old/libhtp-0.5.31/htp/htp_connection_parser.h 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/htp/htp_connection_parser.h 2019-12-13 10:30:17.000000000 +0100 @@ -57,6 +57,7 @@ * @param[in] timestamp Optional. */ void htp_connp_close(htp_connp_t *connp, const htp_time_t *timestamp); +void htp_connp_req_close(htp_connp_t *connp, const htp_time_t *timestamp); /** * Creates a new connection parser using the provided configuration. Because diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/htp/htp_request.c new/libhtp-0.5.32/htp/htp_request.c --- old/libhtp-0.5.31/htp/htp_request.c 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/htp/htp_request.c 2019-12-13 10:30:17.000000000 +0100 @@ -828,40 +828,73 @@ } htp_status_t htp_connp_REQ_FINALIZE(htp_connp_t *connp) { - size_t bytes_left = connp->in_current_len - connp->in_current_read_offset; - - if (bytes_left > 0) { - // If we have more bytes - // Either it is request pipelining - // Or we interpret it as body data - int64_t pos = connp->in_current_read_offset; - int64_t mstart = 0; - // skip past leading whitespace. IIS allows this - while ((pos < connp->in_current_len) && htp_is_space(connp->in_current_data[pos])) - pos++; - if (pos < connp->in_current_len) { - mstart = pos; - // The request method starts at the beginning of the - // line and ends with the first whitespace character. - while ((pos < connp->in_current_len) && (!htp_is_space(connp->in_current_data[pos]))) - pos++; - - int methodi = HTP_M_UNKNOWN; - bstr *method = bstr_dup_mem(connp->in_current_data + mstart, pos - mstart); - if (method) { - methodi = htp_convert_method_to_number(method); - bstr_free(method); - } - if (methodi == HTP_M_UNKNOWN) { - // Interpret remaining bytes as body data - htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); - connp->in_tx->request_progress = HTP_REQUEST_BODY; - connp->in_state = htp_connp_REQ_BODY_IDENTITY; - connp->in_body_data_left = bytes_left; - return HTP_OK; + if (connp->in_status != HTP_STREAM_CLOSED) { + IN_PEEK_NEXT(connp); + if (connp->in_next_byte == -1) { + return htp_tx_state_request_complete(connp->in_tx); + } + if (connp->in_next_byte != LF || connp->in_current_consume_offset >= connp->in_current_read_offset) { + for (;;) {//;i < max_read; i++) { + IN_COPY_BYTE_OR_RETURN(connp); + // Have we reached the end of the line? For some reason + // we can't test after IN_COPY_BYTE_OR_RETURN */ + if (connp->in_next_byte == LF) + break; } } } + + unsigned char *data; + size_t len; + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING request finalize", data, len); +#endif + if (len == 0) { + //closing + return htp_tx_state_request_complete(connp->in_tx); + } + + size_t pos = 0; + size_t mstart = 0; + // skip past leading whitespace. IIS allows this + while ((pos < len) && htp_is_space(data[pos])) + pos++; + if (pos) + mstart = pos; + // The request method starts at the beginning of the + // line and ends with the first whitespace character. + while ((pos < len) && (!htp_is_space(data[pos]))) + pos++; + + if (pos > mstart) { + int methodi = HTP_M_UNKNOWN; + bstr *method = bstr_dup_mem(data + mstart, pos - mstart); + if (method) { + methodi = htp_convert_method_to_number(method); + bstr_free(method); + } + if (methodi == HTP_M_UNKNOWN) { + // Interpret remaining bytes as body data + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); + htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx, data, len); + htp_connp_req_clear_buffer(connp); + return rc; + } + } + //else + //unread last end of line so that REQ_LINE works + if (connp->in_current_read_offset < (int64_t)len) { + connp->in_current_read_offset=0; + } else { + connp->in_current_read_offset-=len; + } + if (connp->in_current_read_offset < connp->in_current_consume_offset) { + connp->in_current_consume_offset=connp->in_current_read_offset; + } + return htp_tx_state_request_complete(connp->in_tx); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/htp/htp_response.c new/libhtp-0.5.32/htp/htp_response.c --- old/libhtp-0.5.31/htp/htp_response.c 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/htp/htp_response.c 2019-12-13 10:30:17.000000000 +0100 @@ -1070,21 +1070,52 @@ } htp_status_t htp_connp_RES_FINALIZE(htp_connp_t *connp) { - int bytes_left = connp->out_current_len - connp->out_current_read_offset; - unsigned char * data = connp->out_current_data + connp->out_current_read_offset; + if (connp->out_status != HTP_STREAM_CLOSED) { + OUT_PEEK_NEXT(connp); + if (connp->out_next_byte == -1) { + return htp_tx_state_response_complete_ex(connp->out_tx, 0); + } + if (connp->out_next_byte != LF || connp->out_current_consume_offset >= connp->out_current_read_offset) { + for (;;) {//;i < max_read; i++) { + OUT_COPY_BYTE_OR_RETURN(connp); + // Have we reached the end of the line? For some reason + // we can't test after IN_COPY_BYTE_OR_RETURN */ + if (connp->out_next_byte == LF) + break; + } + } + } + size_t bytes_left; + unsigned char * data; + + if (htp_connp_res_consolidate_data(connp, &data, &bytes_left) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING response finalize", data, bytes_left); +#endif + if (bytes_left == 0) { + //closing + return htp_tx_state_response_complete_ex(connp->out_tx, 0); + } - if (bytes_left > 0 && - htp_treat_response_line_as_body(data, bytes_left)) { + if (htp_treat_response_line_as_body(data, bytes_left)) { // Interpret remaining bytes as body data htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected response body"); - connp->out_current_read_offset += bytes_left; - connp->out_current_consume_offset += bytes_left; - connp->out_stream_offset += bytes_left; - connp->out_body_data_left -= bytes_left; htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, data, bytes_left); + htp_connp_res_clear_buffer(connp); return rc; } + //unread last end of line so that RES_LINE works + if (connp->out_current_read_offset < (int64_t)bytes_left) { + connp->out_current_read_offset=0; + } else { + connp->out_current_read_offset-=bytes_left; + } + if (connp->out_current_read_offset < connp->out_current_consume_offset) { + connp->out_current_consume_offset=connp->out_current_read_offset; + } return htp_tx_state_response_complete_ex(connp->out_tx, 0 /* not hybrid mode */); } @@ -1112,6 +1143,10 @@ connp->out_tx = htp_list_get(connp->conn->transactions, connp->out_next_tx_index); if (connp->out_tx == NULL) { htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Unable to match response to request"); + // finalize dangling request waiting for next request or body + if (connp->in_state == htp_connp_REQ_FINALIZE) { + htp_tx_state_request_complete(connp->in_tx); + } connp->out_tx = htp_connp_tx_create(connp); if (connp->out_tx == NULL) { return HTP_ERROR; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/test/files/91-request-unexpected-body.t new/libhtp-0.5.32/test/files/91-request-unexpected-body.t --- old/libhtp-0.5.31/test/files/91-request-unexpected-body.t 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/test/files/91-request-unexpected-body.t 2019-12-13 10:30:17.000000000 +0100 @@ -4,6 +4,7 @@ Content-Type: application/x-www-form-urlencoded login=foo&password=bar + <<< HTTP/1.1 200 OK Content-Length: 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/test/files/97-requests-cut.t new/libhtp-0.5.32/test/files/97-requests-cut.t --- old/libhtp-0.5.31/test/files/97-requests-cut.t 1970-01-01 01:00:00.000000000 +0100 +++ new/libhtp-0.5.32/test/files/97-requests-cut.t 2019-12-13 10:30:17.000000000 +0100 @@ -0,0 +1,9 @@ +>>> +GET /?p=%20 HTTP/1.1 +User-Agent: Mozilla + +G +>>> +ET /?p=%21 HTTP/1.1 +User-Agent: Mozilla + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/test/files/98-responses-cut.t new/libhtp-0.5.32/test/files/98-responses-cut.t --- old/libhtp-0.5.31/test/files/98-responses-cut.t 1970-01-01 01:00:00.000000000 +0100 +++ new/libhtp-0.5.32/test/files/98-responses-cut.t 2019-12-13 10:30:17.000000000 +0100 @@ -0,0 +1,26 @@ +>>> +GET /?p=%20 HTTP/1.1 +User-Agent: Mozilla + +GET /?p=%21 HTTP/1.1 +User-Agent: Mozilla + +<<< +HTTP/1.0 200 OK +Date: Mon, 31 Aug 2009 20:25:50 GMT +Server: Apache +Connection: close +Content-Type: text/html +Content-Length: 14 + +Hello World! +H +<<< +TTP/1.0 200 OK +Date: Mon, 31 Aug 2009 20:25:50 GMT +Server: Apache +Connection: close +Content-Type: text/html +Content-Length: 13 + +Hello People! \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.31/test/test_main.cpp new/libhtp-0.5.32/test/test_main.cpp --- old/libhtp-0.5.31/test/test_main.cpp 2019-09-22 10:16:56.000000000 +0200 +++ new/libhtp-0.5.32/test/test_main.cpp 2019-12-13 10:30:17.000000000 +0100 @@ -1483,7 +1483,7 @@ htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); ASSERT_TRUE(tx != NULL); - ASSERT_EQ(1, tx->request_ignored_lines); + /*part of previous request body ASSERT_EQ(1, tx->request_ignored_lines);*/ } TEST_F(ConnectionParsing, PostNoBody) { @@ -1594,7 +1594,7 @@ htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); ASSERT_TRUE(tx != NULL); - ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + //error first ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); ASSERT_EQ(HTP_RESPONSE_HEADERS, tx->response_progress); } @@ -2032,3 +2032,39 @@ ASSERT_EQ(68, tx->response_entity_len); } #endif + +TEST_F(ConnectionParsing, RequestsCut) { + int rc = test_run(home, "97-requests-cut.t", cfg, &connp); + ASSERT_GE(rc, 0); + + ASSERT_EQ(2, htp_list_size(connp->conn->transactions)); + htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + + tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); +} + +TEST_F(ConnectionParsing, ResponsesCut) { + int rc = test_run(home, "98-responses-cut.t", cfg, &connp); + ASSERT_GE(rc, 0); + + ASSERT_EQ(2, htp_list_size(connp->conn->transactions)); + htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(200, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); + + tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(200, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); +}
