Hello community, here is the log from the commit of package libhtp for openSUSE:Factory checked in at 2020-09-15 16:28:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/libhtp (Old) and /work/SRC/openSUSE:Factory/.libhtp.new.4249 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libhtp" Tue Sep 15 16:28:29 2020 rev:5 rq:834279 version:0.5.34 Changes: -------- --- /work/SRC/openSUSE:Factory/libhtp/libhtp.changes 2020-05-01 11:12:34.823746607 +0200 +++ /work/SRC/openSUSE:Factory/.libhtp.new.4249/libhtp.changes 2020-09-15 16:28:32.178610927 +0200 @@ -1,0 +2,8 @@ +Sun Sep 13 13:03:31 UTC 2020 - Martin Hauke <mar...@gmx.de> + +- Update to version 0.5.34 + * support data GAP handling + * support 100-continue Expect + * lzma: give more control over settings + +------------------------------------------------------------------- Old: ---- libhtp-0.5.33.tar.gz New: ---- libhtp-0.5.34.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ libhtp.spec ++++++ --- /var/tmp/diff_new_pack.GCEVqB/_old 2020-09-15 16:28:32.838611559 +0200 +++ /var/tmp/diff_new_pack.GCEVqB/_new 2020-09-15 16:28:32.842611563 +0200 @@ -19,7 +19,7 @@ %define sover 2 %define lname %{name}%{sover} Name: libhtp -Version: 0.5.33 +Version: 0.5.34 Release: 0 Summary: HTTP normalizer and parser License: BSD-3-Clause ++++++ libhtp-0.5.33.tar.gz -> libhtp-0.5.34.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/.travis.yml new/libhtp-0.5.34/.travis.yml --- old/libhtp-0.5.33/.travis.yml 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/.travis.yml 2020-09-10 21:47:54.000000000 +0200 @@ -3,7 +3,7 @@ - gcc - clang # Change this to your needs -script: sh autogen.sh && ./configure && make && make check && make distcheck +script: sh autogen.sh && ./configure && make && (make check || cat test/test-suite.log) && make distcheck before_install: - sudo apt-get update -qq - sudo apt-get install -y build-essential autoconf automake libtool zlib1g zlib1g-dev make diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/ChangeLog new/libhtp-0.5.34/ChangeLog --- old/libhtp-0.5.33/ChangeLog 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/ChangeLog 2020-09-10 21:47:54.000000000 +0200 @@ -1,3 +1,12 @@ +0.5.34 (11 September 2020) +-------------------------- + +- support data GAP handling + +- support 100-continue Expect + +- lzma: give more control over settings + 0.5.33 (27 April 2020) ---------------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/VERSION new/libhtp-0.5.34/VERSION --- old/libhtp-0.5.33/VERSION 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/VERSION 2020-09-10 21:47:54.000000000 +0200 @@ -1,2 +1,2 @@ # This file is intended to be sourced by sh -PKG_VERSION=0.5.33 +PKG_VERSION=0.5.34 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config.c new/libhtp-0.5.34/htp/htp_config.c --- old/libhtp-0.5.33/htp/htp_config.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_config.c 2020-09-10 21:47:54.000000000 +0200 @@ -159,6 +159,7 @@ cfg->extract_request_files_limit = -1; // Use the parser default. cfg->response_decompression_layer_limit = 2; // 2 layers seem fairly common cfg->lzma_memlimit = HTP_LZMA_MEMLIMIT; + cfg->response_lzma_layer_limit = 1; // default is only one layer cfg->compression_bomb_limit = HTP_COMPRESSION_BOMB_LIMIT; cfg->compression_time_limit = HTP_COMPRESSION_TIME_LIMIT_USEC; @@ -515,6 +516,11 @@ cfg->lzma_memlimit = memlimit; } +void htp_config_set_lzma_layers(htp_cfg_t *cfg, int limit) { + if (cfg == NULL) return; + cfg->response_lzma_layer_limit = limit; +} + void htp_config_set_compression_bomb_limit(htp_cfg_t *cfg, size_t bomblimit) { if (cfg == NULL) return; if (bomblimit > INT32_MAX) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config.h new/libhtp-0.5.34/htp/htp_config.h --- old/libhtp-0.5.33/htp/htp_config.h 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_config.h 2020-09-10 21:47:54.000000000 +0200 @@ -435,6 +435,14 @@ void htp_config_set_lzma_memlimit(htp_cfg_t *cfg, size_t memlimit); /** + * Configures the maximum layers LibHTP will pass to liblzma. + * + * @param[in] cfg + * @param[in] limit + */ +void htp_config_set_lzma_layers(htp_cfg_t *cfg, int limit); + +/** * Configures the maximum compression bomb size LibHTP will decompress. * * @param[in] cfg diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config_private.h new/libhtp-0.5.34/htp/htp_config_private.h --- old/libhtp-0.5.33/htp/htp_config_private.h 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_config_private.h 2020-09-10 21:47:54.000000000 +0200 @@ -351,6 +351,9 @@ /** max time for a decompression bomb. */ int32_t compression_time_limit; + + /** How many layers of compression we will decompress (0 => no lzma). */ + int response_lzma_layer_limit; }; #ifdef __cplusplus diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_decompressors.c new/libhtp-0.5.34/htp/htp_decompressors.c --- old/libhtp-0.5.33/htp/htp_decompressors.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_decompressors.c 2020-09-10 21:47:54.000000000 +0200 @@ -439,7 +439,8 @@ switch (format) { case HTP_COMPRESSION_LZMA: - if (connp->cfg->lzma_memlimit > 0) { + if (connp->cfg->lzma_memlimit > 0 && + connp->cfg->response_lzma_layer_limit > 0) { LzmaDec_Construct(&drec->state); } else { htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "LZMA decompression disabled"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_request.c new/libhtp-0.5.34/htp/htp_request.c --- old/libhtp-0.5.33/htp/htp_request.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_request.c 2020-09-10 21:47:54.000000000 +0200 @@ -775,6 +775,10 @@ #ifdef HTP_DEBUG fprint_raw_data(stderr, __func__, data, len); #endif + if (len == 0) { + htp_connp_req_clear_buffer(connp); + return HTP_DATA; + } // Is this a line that should be ignored? if (htp_connp_is_line_ignorable(connp, data, len)) { @@ -886,12 +890,18 @@ bstr_free(method); } if (methodi == HTP_M_UNKNOWN) { + if (connp->in_body_data_left <= 0) { + // log only once per transaction + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); + } else { + connp->in_body_data_left = 1; + } // Interpret remaining bytes as body data - htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx, data, len); htp_connp_req_clear_buffer(connp); return rc; } // else continue + connp->in_body_data_left = -1; } //unread last end of line so that REQ_LINE works if (connp->in_current_read_offset < (int64_t)len) { @@ -992,7 +1002,7 @@ // only if the stream has been closed. We do not allow zero-sized // chunks in the API, but we use them internally to force the parsers // to finalize parsing. - if (((data == NULL) || (len == 0)) && (connp->in_status != HTP_STREAM_CLOSED)) { + if (len == 0 && connp->in_status != HTP_STREAM_CLOSED) { htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data chunks are not allowed"); #ifdef HTP_DEBUG @@ -1045,7 +1055,25 @@ // Return if there's been an error or if we've run out of data. We are relying // on processors to supply error messages, so we'll keep quiet here. - htp_status_t rc = connp->in_state(connp); + + htp_status_t rc; + //handle gap + if (data == NULL && len > 0) { + //cannot switch over a function pointer in C + if (connp->in_state == htp_connp_REQ_BODY_IDENTITY || + connp->in_state == htp_connp_REQ_IGNORE_DATA_AFTER_HTTP_0_9) { + rc = connp->in_state(connp); + } else if (connp->in_state == htp_connp_REQ_FINALIZE) { + //simple version without probing + rc = htp_tx_state_request_complete(connp->in_tx); + } else { + // go to htp_connp_REQ_CONNECT_PROBE_DATA ? + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Gaps are not allowed during this state"); + return HTP_STREAM_CLOSED; + } + } else { + rc = connp->in_state(connp); + } if (rc == HTP_OK) { if (connp->in_status == HTP_STREAM_TUNNEL) { #ifdef HTP_DEBUG diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_response.c new/libhtp-0.5.34/htp/htp_response.c --- old/libhtp-0.5.33/htp/htp_response.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_response.c 2020-09-10 21:47:54.000000000 +0200 @@ -608,6 +608,19 @@ return HTP_OK; } + // A request can indicate it waits for headers validation + // before sending its body cf + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect + if (connp->out_tx->response_status_number >= 400 && + connp->out_tx->response_status_number <= 499 && + connp->in_content_length > 0 && + connp->in_body_data_left == connp->in_content_length) { + htp_header_t *exp = htp_table_get_c(connp->out_tx->request_headers, "expect"); + if ((exp != NULL) && (bstr_cmp_c_nocase(exp->value, "100-continue") == 0)) { + connp->in_state = htp_connp_REQ_FINALIZE; + } + } + // 1. Any response message which MUST NOT include a message-body // (such as the 1xx, 204, and 304 responses and any response to a HEAD // request) is always terminated by the first empty line after the @@ -1228,7 +1241,7 @@ // only if the stream has been closed. We do not allow zero-sized // chunks in the API, but we use it internally to force the parsers // to finalize parsing. - if (((data == NULL) || (len == 0)) && (connp->out_status != HTP_STREAM_CLOSED)) { + if (len == 0 && connp->out_status != HTP_STREAM_CLOSED) { htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data chunks are not allowed"); #ifdef HTP_DEBUG @@ -1277,7 +1290,22 @@ // or if we've run out of data. We are relying // on processors to add error messages, so we'll // keep quiet here. - htp_status_t rc = connp->out_state(connp); + htp_status_t rc; + + //handle gap + if (data == NULL && len > 0) { + if (connp->out_state == htp_connp_RES_BODY_IDENTITY_CL_KNOWN || + connp->out_state == htp_connp_RES_BODY_IDENTITY_STREAM_CLOSE) { + rc = connp->out_state(connp); + } else if (connp->out_state == htp_connp_RES_FINALIZE) { + rc = htp_tx_state_response_complete_ex(connp->out_tx, 0); + } else { + htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Gaps are not allowed during this state"); + return HTP_STREAM_CLOSED; + } + } else { + rc = connp->out_state(connp); + } if (rc == HTP_OK) { if (connp->out_status == HTP_STREAM_TUNNEL) { #ifdef HTP_DEBUG diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_transaction.c new/libhtp-0.5.34/htp/htp_transaction.c --- old/libhtp-0.5.33/htp/htp_transaction.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_transaction.c 2020-09-10 21:47:54.000000000 +0200 @@ -1280,6 +1280,7 @@ } else { int layers = 0; htp_decompressor_t *comp = NULL; + int nblzma = 0; uint8_t *tok = NULL; size_t tok_len = 0; @@ -1326,6 +1327,12 @@ cetype = HTP_COMPRESSION_DEFLATE; } else if (bstr_util_cmp_mem(tok, tok_len, "lzma", 4) == 0) { cetype = HTP_COMPRESSION_LZMA; + nblzma++; + if (nblzma > tx->connp->cfg->response_lzma_layer_limit) { + htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, + "Compression bomb: double lzma encoding"); + break; + } } else if (bstr_util_cmp_mem(tok, tok_len, "inflate", 7) == 0) { cetype = HTP_COMPRESSION_NONE; } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_util.c new/libhtp-0.5.34/htp/htp_util.c --- old/libhtp-0.5.33/htp/htp_util.c 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp/htp_util.c 2020-09-10 21:47:54.000000000 +0200 @@ -1975,7 +1975,12 @@ * */ void fprint_raw_data(FILE *stream, const char *name, const void *data, size_t len) { - fprint_raw_data_ex(stream, name, data, 0, len); + // may happen for gaps + if (data == NULL && len > 0) { + fprintf(stream, "\n%s: ptr NULL len %u\n", name, (unsigned int)len); + } else { + fprint_raw_data_ex(stream, name, data, 0, len); + } } /** diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/htp.pc.in new/libhtp-0.5.34/htp.pc.in --- old/libhtp-0.5.33/htp.pc.in 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/htp.pc.in 2020-09-10 21:47:54.000000000 +0200 @@ -7,6 +7,6 @@ Description: A security-aware HTTP parser, designed for use in IDS/IPS and WAF products. Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lhtp -Libs.private: @LIBICONV@ +Libs.private: -lz @LIBICONV@ Cflags: -I${includedir} -I${libdir}/htp/include diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/test/files/99-expect-100.t new/libhtp-0.5.34/test/files/99-expect-100.t --- old/libhtp-0.5.33/test/files/99-expect-100.t 1970-01-01 01:00:00.000000000 +0100 +++ new/libhtp-0.5.34/test/files/99-expect-100.t 2020-09-10 21:47:54.000000000 +0200 @@ -0,0 +1,26 @@ +>>> +PUT /forbidden HTTP/1.1 +Content-Length: 14 +Expect: 100-continue + + +<<< +HTTP/1.0 401 Forbidden +Content-Length: 0 + + +>>> +POST /ok HTTP/1.1 +Content-Length: 14 +Expect: 100-continue + + +<<< +HTTP/1.0 100 continue + + +>>> +Hello People! + +<<< +HTTP/1.0 200 OK diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libhtp-0.5.33/test/test_main.cpp new/libhtp-0.5.34/test/test_main.cpp --- old/libhtp-0.5.33/test/test_main.cpp 2020-04-27 15:30:04.000000000 +0200 +++ new/libhtp-0.5.34/test/test_main.cpp 2020-09-10 21:47:54.000000000 +0200 @@ -2068,3 +2068,23 @@ ASSERT_EQ(200, tx->response_status_number); ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); } + +TEST_F(ConnectionParsing, Expect100) { + int rc = test_run(home, "99-expect-100.t", cfg, &connp); + ASSERT_GE(rc, 0); + + ASSERT_EQ(2, htp_list_size(connp->conn->transactions)); + htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "PUT")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(401, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); + + tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "POST")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(200, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); +}