michael-o commented on a change in pull request #206: URL: https://github.com/apache/httpcomponents-core/pull/206#discussion_r464058849
########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,78 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.impl.io; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.io.HttpClientConnection; +import org.apache.hc.core5.http.io.ResponseOutOfOrderStrategy; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Default {@link ResponseOutOfOrderStrategy} implementation which checks for premature responses every 8 KiB based + * on testing using values between 4 KiB and 128 KiB. This is optimized for correctness and results in a maximum + * upload speed of 8 MiB/s. + * + * @since 5.1 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class DefaultResponseOutOfOrderStrategy implements ResponseOutOfOrderStrategy { + + public static final DefaultResponseOutOfOrderStrategy INSTANCE = new DefaultResponseOutOfOrderStrategy(); + + private static final int CHUNK_SIZE = 8 * 1024; + + @Override + public boolean checkForEarlyResponse( + final HttpClientConnection connection, + final ClassicHttpRequest request, + final InputStream socketInputStream, + final long totalBytesSent, + final long nextWriteSize) throws IOException { + if (totalBytesSent == 0 || nextWriteBeginsNewChunk(totalBytesSent, nextWriteSize)) { + final boolean ssl = connection.getSSLSession() != null; + return ssl ? connection.isDataAvailable(Timeout.ONE_MILLISECOND) : (socketInputStream.available() > 0); + } + return false; + } + + private static boolean nextWriteBeginsNewChunk(final long totalBytesSent, final long nextWriteSize) { + final long currentChunkIndex = totalBytesSent / CHUNK_SIZE; + final long newChunkIndex = (totalBytesSent + nextWriteSize) / CHUNK_SIZE; + return currentChunkIndex < newChunkIndex; + } + + @Override + public String toString() { Review comment: I think this one is redundant unless chunk size is configurable and can be printed. ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,78 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.impl.io; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.io.HttpClientConnection; +import org.apache.hc.core5.http.io.ResponseOutOfOrderStrategy; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Default {@link ResponseOutOfOrderStrategy} implementation which checks for premature responses every 8 KiB based + * on testing using values between 4 KiB and 128 KiB. This is optimized for correctness and results in a maximum + * upload speed of 8 MiB/s. + * + * @since 5.1 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class DefaultResponseOutOfOrderStrategy implements ResponseOutOfOrderStrategy { + + public static final DefaultResponseOutOfOrderStrategy INSTANCE = new DefaultResponseOutOfOrderStrategy(); + + private static final int CHUNK_SIZE = 8 * 1024; + + @Override + public boolean checkForEarlyResponse( + final HttpClientConnection connection, + final ClassicHttpRequest request, + final InputStream socketInputStream, + final long totalBytesSent, + final long nextWriteSize) throws IOException { + if (totalBytesSent == 0 || nextWriteBeginsNewChunk(totalBytesSent, nextWriteSize)) { Review comment: Can `totalBytesSent == 0` actually happen? Does `totalBytes` not contain the length of the headers? ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultBHttpClientConnection.java ########## @@ -157,41 +198,40 @@ public void sendRequestEntity(final ClassicHttpRequest request) throws HttpExcep try (final OutputStream outStream = createContentOutputStream( len, this.outbuffer, new OutputStream() { - final boolean ssl = socketHolder.getSocket() instanceof SSLSocket; - final InputStream socketInputStream = socketHolder.getInputStream(); final OutputStream socketOutputStream = socketHolder.getOutputStream(); + final InputStream socketInputStream = socketHolder.getInputStream(); long totalBytes = 0; - long chunks = -1; - void checkForEarlyResponse() throws IOException { - final long n = totalBytes / (8 * 1024); - if (n > chunks) { - chunks = n; - if (ssl ? isDataAvailable(Timeout.ONE_MILLISECOND) : (socketInputStream.available() > 0)) { - throw new ResponseOutOfOrderException(); - } + void checkForEarlyResponse(final long totalBytesSent, final int nextWriteSize) throws IOException { + if (responseOutOfOrderStrategy.checkForEarlyResponse( + DefaultBHttpClientConnection.this, + request, + socketInputStream, + totalBytesSent, + nextWriteSize)) { + throw new ResponseOutOfOrderException(); } } @Override public void write(final byte[] b) throws IOException { + checkForEarlyResponse(totalBytes, b.length); Review comment: Can you explain why the check now happens before the write in general? Previously, the the headers and the first chunk already exceeded the chunk size the server could deny the request and we were able to detect that. Now, the total header size has to be at least `CHUNK_SIZE` or similar. ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,78 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.impl.io; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.io.HttpClientConnection; +import org.apache.hc.core5.http.io.ResponseOutOfOrderStrategy; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Default {@link ResponseOutOfOrderStrategy} implementation which checks for premature responses every 8 KiB based + * on testing using values between 4 KiB and 128 KiB. This is optimized for correctness and results in a maximum + * upload speed of 8 MiB/s. + * + * @since 5.1 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class DefaultResponseOutOfOrderStrategy implements ResponseOutOfOrderStrategy { + + public static final DefaultResponseOutOfOrderStrategy INSTANCE = new DefaultResponseOutOfOrderStrategy(); + + private static final int CHUNK_SIZE = 8 * 1024; + + @Override + public boolean checkForEarlyResponse( Review comment: I think the name is a bit misleading. Since it is a boolean and used as such with an exception, shouldn't it be `isEarlyResponse` or similar? ```java if (isEarlyResponse()) throw new ...Exception(); ``` ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,78 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.impl.io; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.io.HttpClientConnection; +import org.apache.hc.core5.http.io.ResponseOutOfOrderStrategy; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Default {@link ResponseOutOfOrderStrategy} implementation which checks for premature responses every 8 KiB based + * on testing using values between 4 KiB and 128 KiB. This is optimized for correctness and results in a maximum + * upload speed of 8 MiB/s. + * + * @since 5.1 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class DefaultResponseOutOfOrderStrategy implements ResponseOutOfOrderStrategy { + + public static final DefaultResponseOutOfOrderStrategy INSTANCE = new DefaultResponseOutOfOrderStrategy(); + + private static final int CHUNK_SIZE = 8 * 1024; + + @Override + public boolean checkForEarlyResponse( + final HttpClientConnection connection, + final ClassicHttpRequest request, + final InputStream socketInputStream, + final long totalBytesSent, + final long nextWriteSize) throws IOException { + if (totalBytesSent == 0 || nextWriteBeginsNewChunk(totalBytesSent, nextWriteSize)) { + final boolean ssl = connection.getSSLSession() != null; + return ssl ? connection.isDataAvailable(Timeout.ONE_MILLISECOND) : (socketInputStream.available() > 0); + } + return false; + } + + private static boolean nextWriteBeginsNewChunk(final long totalBytesSent, final long nextWriteSize) { Review comment: Shouldn't the name rather be: `nextWriteStartsNewChunk` or `nextWriteBeginsWithNewChunk`? ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/io/ResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,64 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.io; + +import org.apache.hc.core5.http.ClassicHttpRequest; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Represents a strategy to determine how frequently the client should check for an out of order response. + * An out of order response is sent before the server has read the full request. If the client fails to + * check for an early response then a {@link java.net.SocketException} or {@link java.net.SocketTimeoutException} + * may be thrown while writing the request entity after a timeout is reached on either the client or server. + * + * @since 5.1 + */ +public interface ResponseOutOfOrderStrategy { + + /** + * Called before each write to the to a socket {@link java.io.OutputStream} with the number of + * bytes that have already been sent, and the size of the write that will occur if this check + * does not encounter an out of order response. + * + * @param connection The connection used to send the current request. + * @param request The current request. + * @param socketInputStream The response stream, this may be used to check for an early response. Review comment: two spaces between `used` and `to`. ########## File path: httpcore5/src/main/java/org/apache/hc/core5/http/impl/io/DefaultResponseOutOfOrderStrategy.java ########## @@ -0,0 +1,78 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + * + */ + +package org.apache.hc.core5.http.impl.io; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.io.HttpClientConnection; +import org.apache.hc.core5.http.io.ResponseOutOfOrderStrategy; +import org.apache.hc.core5.util.Timeout; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Default {@link ResponseOutOfOrderStrategy} implementation which checks for premature responses every 8 KiB based + * on testing using values between 4 KiB and 128 KiB. This is optimized for correctness and results in a maximum + * upload speed of 8 MiB/s. + * + * @since 5.1 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public final class DefaultResponseOutOfOrderStrategy implements ResponseOutOfOrderStrategy { Review comment: When have strategies which provide non-default constructors for modification of behavior. Should we add a single args constructor with a preferred chunk size? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
