http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/core/io/src/main/java/org/apache/zest/io/Transforms.java ---------------------------------------------------------------------- diff --git a/core/io/src/main/java/org/apache/zest/io/Transforms.java b/core/io/src/main/java/org/apache/zest/io/Transforms.java deleted file mode 100644 index 9cec3c5..0000000 --- a/core/io/src/main/java/org/apache/zest/io/Transforms.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * - */ - -package org.apache.zest.io; - -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.text.MessageFormat; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.logging.Logger; - -/** - * Utility class for I/O transforms - */ -public class Transforms -{ - /** - * Filter items in a transfer by applying the given Specification to each item. - * - * @param specification The Specification defining the items to not filter away. - * @param output The Output instance to receive to result. - * @param <T> The item type - * @param <Receiver2ThrowableType> Exception type that might be thrown by the Receiver. - * - * @return And Output encapsulation the filter operation. - */ - public static <T, Receiver2ThrowableType extends Throwable> Output<T, Receiver2ThrowableType> filter( final Predicate<? super T> specification, - final Output<T, Receiver2ThrowableType> output - ) - { - return new Output<T, Receiver2ThrowableType>() - { - @Override - public <SenderThrowableType extends Throwable> void receiveFrom( final Sender<? extends T, SenderThrowableType> sender ) - throws Receiver2ThrowableType, SenderThrowableType - { - output.receiveFrom( new Sender<T, SenderThrowableType>() - { - @Override - public <ReceiverThrowableType extends Throwable> void sendTo( final Receiver<? super T, ReceiverThrowableType> receiver ) - throws ReceiverThrowableType, SenderThrowableType - { - sender.sendTo( new Receiver<T, ReceiverThrowableType>() - { - @Override - public void receive( T item ) - throws ReceiverThrowableType - { - if( specification.test( item ) ) - { - receiver.receive( item ); - } - } - } ); - } - } ); - } - }; - } - - /** - * Map items in a transfer from one type to another by applying the given function. - * - * @param function The transformation function to apply to the streaming items. - * @param output The output to receive the transformed items. - * @param <From> The type of the incoming items. - * @param <To> The type of the transformed items. - * @param <Receiver2ThrowableType> The exception type that the Receiver might throw. - * - * @return An Output instance that encapsulates the map transformation. - */ - public static <From, To, Receiver2ThrowableType extends Throwable> Output<From, Receiver2ThrowableType> map( final Function<? super From, ? extends To> function, - final Output<To, Receiver2ThrowableType> output - ) - { - return new Output<From, Receiver2ThrowableType>() - { - @Override - public <SenderThrowableType extends Throwable> void receiveFrom( final Sender<? extends From, SenderThrowableType> sender ) - throws Receiver2ThrowableType, SenderThrowableType - { - output.receiveFrom( new Sender<To, SenderThrowableType>() - { - @Override - public <ReceiverThrowableType extends Throwable> void sendTo( final Receiver<? super To, ReceiverThrowableType> receiver ) - throws ReceiverThrowableType, SenderThrowableType - { - sender.sendTo( new Receiver<From, ReceiverThrowableType>() - { - @Override - public void receive( From item ) - throws ReceiverThrowableType - { - receiver.receive( function.apply( item ) ); - } - } ); - } - } ); - } - }; - } - - /** - * Apply the given function to items in the transfer that match the given specification. Other items will pass - * through directly. - * - * @param specification The Specification defining which items should be transformed. - * @param function The transformation function. - * @param output The Output that will receive the resulting items. - * @param <T> The item type. Items can not be transformed to a new type. - * @param <Receiver2ThrowableType> The exception that the Receiver might throw. - * - * @return An Output instance that encapsulates the operation. - */ - public static <T, Receiver2ThrowableType extends Throwable> Output<T, Receiver2ThrowableType> filteredMap( final Predicate<? super T> specification, - final Function<? super T, ? extends T> function, - final Output<T, Receiver2ThrowableType> output - ) - { - return new Output<T, Receiver2ThrowableType>() - { - @Override - public <SenderThrowableType extends Throwable> void receiveFrom( final Sender<? extends T, SenderThrowableType> sender ) - throws Receiver2ThrowableType, SenderThrowableType - { - output.receiveFrom( new Sender<T, SenderThrowableType>() - { - @Override - public <ReceiverThrowableType extends Throwable> void sendTo( final Receiver<? super T, ReceiverThrowableType> receiver ) - throws ReceiverThrowableType, SenderThrowableType - { - sender.sendTo( new Receiver<T, ReceiverThrowableType>() - { - @Override - public void receive( T item ) - throws ReceiverThrowableType - { - if( specification.test( item ) ) - { - receiver.receive( function.apply( item ) ); - } - else - { - receiver.receive( item ); - } - } - } ); - } - } ); - } - }; - } - - /** - * Wrapper for Outputs that uses a lock whenever a transfer is instantiated. Typically a read-lock would be used on - * the sending side and a write-lock would be used on the receiving side. Inputs can use this as well to create a - * wrapper on the send side when transferTo is invoked. - * - * @param lock the lock to be used for transfers - * @param output output to be wrapped - * @param <T> The Item type - * @param <Receiver2ThrowableType> The Exception type that the Receiver might throw. - * - * @return Output wrapper that uses the given lock during transfers. - */ - public static <T, Receiver2ThrowableType extends Throwable> Output<T, Receiver2ThrowableType> lock( final Lock lock, - final Output<T, Receiver2ThrowableType> output - ) - { - return new Output<T, Receiver2ThrowableType>() - { - @Override - public <SenderThrowableType extends Throwable> void receiveFrom( Sender<? extends T, SenderThrowableType> sender ) - throws Receiver2ThrowableType, SenderThrowableType - { - /** - * Fix for this bug: - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6822370 - */ - while( true ) - { - try - { - //noinspection StatementWithEmptyBody - while( !lock.tryLock( 1000, TimeUnit.MILLISECONDS ) ) - { - // On timeout, try again - } - break; // Finally got a lock - } - catch( InterruptedException e ) - { - // Try again - } - } - - try - { - output.receiveFrom( sender ); - } - finally - { - lock.unlock(); - } - } - }; - } - - /** - * Wrapper for Outputs that uses a lock whenever a transfer is instantiated. Typically a read-lock would be used on the sending side and a write-lock - * would be used on the receiving side. - * - * @param lock the lock to be used for transfers - * @param input input to be wrapped - * @param <T> The item type. - * @param <SenderThrowableType> The Exception type that the Sender might throw. - * - * @return Input wrapper that uses the given lock during transfers. - */ - public static <T, SenderThrowableType extends Throwable> Input<T, SenderThrowableType> lock( final Lock lock, - final Input<T, SenderThrowableType> input - ) - { - return new Input<T, SenderThrowableType>() - { - @Override - public <ReceiverThrowableType extends Throwable> void transferTo( Output<? super T, ReceiverThrowableType> output ) - throws SenderThrowableType, ReceiverThrowableType - { - /** - * Fix for this bug: - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6822370 - */ - while( true ) - { - try - { - //noinspection StatementWithEmptyBody - while( !( lock.tryLock() || lock.tryLock( 1000, TimeUnit.MILLISECONDS ) ) ) - { - // On timeout, try again - } - break; // Finally got a lock - } - catch( InterruptedException e ) - { - // Try again - } - } - - try - { - input.transferTo( output ); - } - finally - { - lock.unlock(); - } - } - }; - } - - /** - * Count the number of items in the transfer. - * - * @param <T> - */ - // START SNIPPET: counter - public static class Counter<T> - implements Function<T, T> - { - private volatile long count = 0; - - public long count() - { - return count; - } - - @Override - public T apply( T t ) - { - count++; - return t; - } - } - // END SNIPPET: counter - - /** - * Convert strings to bytes using the given CharSet - */ - @SuppressWarnings( "UnusedDeclaration" ) - public static class String2Bytes - implements Function<String, byte[]> - { - private Charset charSet; - - public String2Bytes( Charset charSet ) - { - this.charSet = charSet; - } - - @Override - public byte[] apply( String s ) - { - return s.getBytes( charSet ); - } - } - - /** - * Convert ByteBuffers to Strings using the given CharSet - */ - public static class ByteBuffer2String - implements Function<ByteBuffer, String> - { - private Charset charSet; - - public ByteBuffer2String( Charset charSet ) - { - this.charSet = charSet; - } - - @Override - public String apply( ByteBuffer buffer ) - { - return new String( buffer.array(), charSet ); - } - } - - /** - * Convert objects to Strings using .toString() - */ - @SuppressWarnings( "UnusedDeclaration" ) - public static class ObjectToString - implements Function<Object, String> - { - @Override - public String apply( Object o ) - { - return o.toString(); - } - } - - /** - * Log the toString() representation of transferred items to the given log. The string is first formatted using MessageFormat - * with the given format. - * - * @param <T> - */ - public static class Log<T> - implements Function<T, T> - { - private Logger logger; - private MessageFormat format; - - public Log( Logger logger, String format ) - { - this.logger = logger; - this.format = new MessageFormat( format ); - } - - @Override - public T apply( T item ) - { - logger.info( format.format( new String[]{ item.toString() } ) ); - return item; - } - } - - /** - * Track progress of transfer by emitting a log message in given intervals. - * - * If logger or format is null, then you need to override the logProgress to do something - * - * @param <T> type of items to be transferred - */ - // START SNIPPET: progress - public static class ProgressLog<T> - implements Function<T, T> - { - private Counter<T> counter; - private Log<String> log; - private final long interval; - - public ProgressLog( Logger logger, String format, long interval ) - { - this.interval = interval; - if( logger != null && format != null ) - { - log = new Log<>( logger, format ); - } - counter = new Counter<>(); - } - - public ProgressLog( long interval ) - { - this.interval = interval; - counter = new Counter<>(); - } - - @Override - public T apply( T t ) - { - counter.apply( t ); - if( counter.count % interval == 0 ) - { - logProgress(); - } - return t; - } - - // Override this to do something other than logging the progress - protected void logProgress() - { - if( log != null ) - { - log.apply( counter.count + "" ); - } - } - } - // END SNIPPET: progress -}
http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/core/io/src/main/java/org/apache/zest/io/package.html ---------------------------------------------------------------------- diff --git a/core/io/src/main/java/org/apache/zest/io/package.html b/core/io/src/main/java/org/apache/zest/io/package.html deleted file mode 100644 index 67d12b4..0000000 --- a/core/io/src/main/java/org/apache/zest/io/package.html +++ /dev/null @@ -1,24 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - ~ - ~ - --> -<html> - <body> - <h2>I/O API.</h2> - </body> -</html> http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/core/io/src/test/java/org/apache/zest/io/InputOutputTest.java ---------------------------------------------------------------------- diff --git a/core/io/src/test/java/org/apache/zest/io/InputOutputTest.java b/core/io/src/test/java/org/apache/zest/io/InputOutputTest.java deleted file mode 100644 index 28d52b0..0000000 --- a/core/io/src/test/java/org/apache/zest/io/InputOutputTest.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * - */ -package org.apache.zest.io; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.Writer; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.rmi.RemoteException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.logging.Logger; -import org.hamcrest.CoreMatchers; -import org.junit.Assert; -import org.junit.Test; -import org.apache.zest.functional.Visitor; - -import static java.util.Arrays.asList; -import static org.apache.zest.io.Inputs.text; -import static org.apache.zest.io.Transforms.lock; -import static org.apache.zest.test.util.Assume.assumeConnectivity; - -/** - * Test Input/Output. - */ -public class InputOutputTest -{ - @Test - public void testCopyFileNoAPI() - throws IOException - { - File source = sourceFile(); - File destination = File.createTempFile( "test", ".txt" ); - destination.deleteOnExit(); - - BufferedReader reader = new BufferedReader( new FileReader( source ) ); - long count = 0; - try - { - BufferedWriter writer = new BufferedWriter( new FileWriter( destination ) ); - try - { - String line; - while( ( line = reader.readLine() ) != null ) - { - count++; - writer.append( line ).append( '\n' ); - } - writer.close(); - } - catch( IOException e ) - { - writer.close(); - destination.delete(); - } - } - finally - { - reader.close(); - } - System.out.println( count ); - } - - @Test - public void testInputOutput() - throws IOException - { - URL source = getClass().getResource( "/iotest.txt" ); - File destination = File.createTempFile( "test", ".txt" ); - destination.deleteOnExit(); - text( source ).transferTo( Outputs.text( destination ) ); - } - - @Test - public void testCopyFile() - throws IOException - { - File source = sourceFile(); - File tempFile = File.createTempFile( "test", ".txt" ); - tempFile.deleteOnExit(); - - Inputs.byteBuffer( source, 1024 ).transferTo( Outputs.byteBuffer( tempFile ) ); - - Assert.assertThat( tempFile.length(), CoreMatchers.equalTo( source.length() ) ); - } - - @Test - public void testCopyURL() - throws IOException - { - assumeConnectivity( "www.google.com", 80 ); - - File tempFile = File.createTempFile( "test", ".txt" ); - tempFile.deleteOnExit(); - - Inputs.text( new URL( "http://www.google.com"; ) ).transferTo( Outputs.text( tempFile ) ); - -// Uncomment to check output Inputs.text( tempFile ).transferTo( Outputs.systemOut() ); - } - - @Test - public void testCopyFileStreams() - throws IOException - { - File source = sourceFile(); - File tempFile = File.createTempFile( "test", ".txt" ); - tempFile.deleteOnExit(); - - Inputs.byteBuffer( new FileInputStream( source ), 1024 ).transferTo( - Outputs.byteBuffer( new FileOutputStream( tempFile ) ) ); - - Assert.assertThat( tempFile.length(), CoreMatchers.equalTo( source.length() ) ); - } - - @Test - public void testLog() - throws IOException - { - File source = sourceFile(); - - text( source ).transferTo( - Transforms.map( new Transforms.Log<String>( Logger.getLogger( getClass().getName() ), "Line: {0}" ), - Outputs.<String>noop() ) ); - } - - @Test - public void testProgressLog() - throws Throwable - { - Integer[] data = new Integer[ 105 ]; - Arrays.fill( data, 42 ); - - Inputs.iterable( Arrays.asList( data ) ).transferTo( - Transforms.map( - new Transforms.ProgressLog<>( - Logger.getLogger( InputOutputTest.class.getName() ), "Data transferred: {0}", 10 ), - Outputs.noop() ) ); - } - - @Test - public void testTextInputsOutputs() - throws IOException - { - File tempFile = File.createTempFile( "test", ".txt" ); - tempFile.deleteOnExit(); - File sourceFile = sourceFile(); - Transforms.Counter<String> stringCounter = new Transforms.Counter<>(); - text( sourceFile ).transferTo( - Transforms.map( - stringCounter, - Transforms.map( new Function<String, String>() - { - public String apply( String s ) - { - System.out.println( s ); - return s; - } - }, Outputs.text( tempFile ) ) - ) - ); - - Assert.assertThat( tempFile.length(), CoreMatchers.equalTo( sourceFile.length() ) ); - Assert.assertThat( stringCounter.count(), CoreMatchers.equalTo( 4L ) ); - } - - @Test - public void testCombineInputs() - throws IOException - { - File tempFile = File.createTempFile( "test", ".txt" ); - tempFile.deleteOnExit(); - File sourceFile = sourceFile(); - Transforms.Counter<String> stringCounter = new Transforms.Counter<>(); - Input<String, IOException> text1 = text( sourceFile ); - Input<String, IOException> text2 = text( sourceFile ); - List<Input<String, IOException>> list = createList( text1, text2 ); - Inputs.combine( list ).transferTo( - Transforms.map( - stringCounter, - Transforms.map( new Function<String, String>() - { - public String apply( String s ) - { - System.out.println( s ); - return s; - } - }, Outputs.text( tempFile ) ) - ) - ); - - Assert.assertThat( tempFile.length(), CoreMatchers.equalTo( sourceFile.length() * 2 ) ); - Assert.assertThat( stringCounter.count(), CoreMatchers.equalTo( 8L ) ); - } - - @SuppressWarnings( "unchecked" ) - private List<Input<String, IOException>> createList( Input<String, IOException> text1, - Input<String, IOException> text2 - ) - { - return asList( text1, text2 ); - } - - @Test( expected = IOException.class ) - public void testInputOutputOutputException() - throws IOException - { - - text( sourceFile() ). - transferTo( writerOutput( new Writer() - { - @Override - public void write( char[] cbuf, int off, int len ) - throws IOException - { - throw new IOException(); - } - - @Override - public void flush() - throws IOException - { - throw new IOException(); - } - - @Override - public void close() - throws IOException - { - throw new IOException(); - } - } ) ); - } - - @Test( expected = RemoteException.class ) - public void testInputOutputInputException() - throws IOException - { - - Input<String, RemoteException> input = new Input<String, RemoteException>() - { - @Override - public <OutputThrowableType extends Throwable> void transferTo( Output<? super String, OutputThrowableType> output ) - throws RemoteException, OutputThrowableType - { - output.receiveFrom( new Sender<String, RemoteException>() - { - @Override - public <ReceiverThrowableType extends Throwable> void sendTo( Receiver<? super String, ReceiverThrowableType> receiverThrowableTypeReceiver ) - throws ReceiverThrowableType, RemoteException - { - throw new RemoteException(); - } - } ); - } - }; - - input.transferTo( - Transforms.map( - new Transforms.Log<String>( Logger.getLogger( getClass().getName() ), "Line: {0}" ), - Outputs.systemOut() - ) - ); - } - - @Test - public void testLock() - throws IOException - { - Lock inputLock = new ReentrantLock(); - Lock outputLock = new ReentrantLock(); - - URL source = getClass().getResource( "/iotest.txt" ); - File destination = File.createTempFile( "test", ".txt" ); - destination.deleteOnExit(); - lock( inputLock, text( source ) ).transferTo( lock( outputLock, Outputs.text( destination ) ) ); - } - - @Test - public void testGenerics() - { - ArrayList<Object> objects = new ArrayList<>( 3 ); - Inputs.iterable( Arrays.asList( "Foo", "Bar", "Xyzzy" ) ).transferTo( Outputs.collection( objects ) ); - - Inputs.iterable( objects ).transferTo( Outputs.systemOut() ); - } - - @Test - public void testOutputstreamInput() - throws Throwable - { - Input<ByteBuffer, IOException> input = Inputs.output( new Visitor<OutputStream, IOException>() - { - @Override - public boolean visit( OutputStream visited ) - throws IOException - { - try( PrintWriter writer = new PrintWriter( visited ) ) - { - writer.print( "Hello World!" ); - } - return true; - } - }, 256 ); - - input.transferTo( Transforms.map( new Transforms.ByteBuffer2String( Charset.defaultCharset() ), Outputs.systemOut() ) ); - input.transferTo( Transforms.map( new Transforms.ByteBuffer2String( Charset.defaultCharset() ), Outputs.systemOut() ) ); - } - - public Output<String, IOException> writerOutput( final Writer writer ) - { - return new Output<String, IOException>() - { - @Override - public <SenderThrowableType extends Throwable> void receiveFrom( Sender<? extends String, SenderThrowableType> sender ) - throws IOException, SenderThrowableType - { - // Here we initiate the transfer - System.out.println( "Open output" ); - final StringBuilder builder = new StringBuilder(); - try - { - sender.sendTo( new Receiver<String, IOException>() - { - @Override - public void receive( String item ) - throws IOException - { - System.out.println( "Receive input" ); - - // Here we can do batch writes if needed - builder.append( item ).append( "\n" ); - } - } ); - - // If transfer went well, do something with it - writer.write( builder.toString() ); - writer.flush(); - System.out.println( "Output written" ); - } - catch( IOException e ) - { - // If transfer failed, potentially rollback writes - System.out.println( "Input failed" ); - throw e; - } - } - }; - } - - private File sourceFile() - { - String path = getClass().getResource( "/iotest.txt" ).getFile(); - return new File( path.replaceAll( "%20", " " ) ); - } -} http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/core/io/src/test/java/org/apache/zest/io/docsupport/IoDocs.java ---------------------------------------------------------------------- diff --git a/core/io/src/test/java/org/apache/zest/io/docsupport/IoDocs.java b/core/io/src/test/java/org/apache/zest/io/docsupport/IoDocs.java deleted file mode 100644 index d360387..0000000 --- a/core/io/src/test/java/org/apache/zest/io/docsupport/IoDocs.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * - */ -package org.apache.zest.io.docsupport; - -import java.io.File; -import java.io.IOException; -import org.apache.zest.io.Inputs; -import org.apache.zest.io.Outputs; - -// START SNIPPET: io2 -import org.apache.zest.io.Transforms.Counter; -import static org.apache.zest.io.Transforms.map; -// END SNIPPET: io2 - -public class IoDocs -{ - public static void main( String[] args ) - throws IOException - { - { -// START SNIPPET: io1 - File source = new File( "source.txt" ); - File destination = new File( "destination.txt" ); - Inputs.text( source ).transferTo( Outputs.text( destination ) ); -// END SNIPPET: io1 - } - { -// START SNIPPET: io2 - File source = new File( "source.txt" ); - File destination = new File( "destination.txt" ); - Counter<String> counter = new Counter<String>(); - Inputs.text( source ).transferTo( map(counter, Outputs.text(destination) )); - System.out.println( "Lines: " + counter.count() ); -// END SNIPPET: io2 - } - } -} http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/core/io/src/test/resources/iotest.txt ---------------------------------------------------------------------- diff --git a/core/io/src/test/resources/iotest.txt b/core/io/src/test/resources/iotest.txt deleted file mode 100644 index a6d84a8..0000000 --- a/core/io/src/test/resources/iotest.txt +++ /dev/null @@ -1,4 +0,0 @@ -Test -of transfer -from input -to output http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/manual/src/docs/tutorials/howto-use-io.txt ---------------------------------------------------------------------- diff --git a/manual/src/docs/tutorials/howto-use-io.txt b/manual/src/docs/tutorials/howto-use-io.txt deleted file mode 100644 index af22317..0000000 --- a/manual/src/docs/tutorials/howto-use-io.txt +++ /dev/null @@ -1,329 +0,0 @@ -////////////////////// - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -////////////////////// - -[[howto-use-io,Use I/O API]] -= Use I/O API = - -NOTE: This article was written on Rickard Ãberg's blog, 6 Nov 2010 - - -The past week I've had to deal with a lot of data shuffling, both in raw form as bytes and strings, and as SPI and -domain level objects. What struck me is that it is notoriously hard to shuffle things from one place to another in a -way that is scalable, performant and handles errors correctly. And I had to do some things over and over again, like -reading strings from files. - -So the thought occurred: there must be a general pattern to how this thing works, which can be extracted and put into a -library. "Reading lines from a text file" should only have to be done once, and then used in whatever scenario requires -it. Let's take a look at a typical example of reading from one file and writing to another to see if we can find out -what the possible pieces could be: - -[source,java] -------------- -1: File source = new File( getClass().getResource( "/iotest.txt" ).getFile() ); -1: File destination = File.createTempFile( "test", ".txt" ); -1: destination.deleteOnExit(); -2: BufferedReader reader = new BufferedReader(new FileReader(source)); -3: long count = 0; -2: try -2: { -4: BufferedWriter writer = new BufferedWriter(new FileWriter(destination)); -4: try -4: { -2: String line = null; -2: while ((line = reader.readLine()) != null) -2: { -3: count++; -4: writer.append( line ).append( '\n' ); -2: } -4: writer.close(); -4: } catch (IOException e) -4: { -4: writer.close(); -4: destination.delete(); -4: } -2: } finally -2: { -2: reader.close(); -2: } -1: System.out.println(count) -------------- - -As the numbers to the left indicates, I've identified four parts in this type of code that could be separated from -each other. - -1) is the client code that initiates a transfer, and which have to know the input and output source. - -2) is the code that reads lines from an input. - -3) is helper code that I use to keep track of what's going on, and which I'd like to reuse no matter what kind of -transfer is being done. - -4) receives the data and writes it down. In this code, if I wanted to implement batching on the read and write side I -could do so by changing the 2 and 4 parts to read/write multiple lines at a time. - -== The API == - -If you want to reproduce what's explained in this tutorial, remember to depend on the Core Runtime artifact that depends -on Core API, Core SPI, Core Bootstrap and Core Functional & I/O APIs: - -include::../../../../core/runtime/build/docs/buildinfo/artifact.txt[] - -See the <<howto-depend-on-zest>> tutorial for details. - -Once theses parts were identified it was mostly just a matter of putting interfaces on these pieces, and making sure -they can be easily used in many different situations. The result is as follows. - -To start with we have Input: - -[snippet,java] --------------- -source=core/io/src/main/java/org/apache/zest/io/Input.java -tag=input -------------- - -Inputs, like Iterables, can be used over and over again to initiate transfers of data from one place to another, in -this case an Output. Since I want this to be generic the type of things that is sent is T, so can be anything -(byte[], String, EntityState, MyDomainObject). I also want the sender and receiver of data to be able to throw their -own exceptions, and this is marked by declaring these as generic exception types. For example, the input may want to -throw SQLException and the output IOException, if anything goes wrong. This should be strongly typed, and both sender -and receiver must know when either side screws up, so that they can recover properly and close any resources they have -opened. - -On the receiving side we then have Output: - -[snippet,java] --------------- -source=core/io/src/main/java/org/apache/zest/io/Output.java -tag=output -------------- - -When receiveFrom is invoked by an Input, as a result of invoking transferTo on the Input, the Output should open -whatever resources it needs to write to, and then expect data to come from a Sender. Both the Input and Output must -have the same type T, so that they agree on what is being sent. We will see later how this can be handled if this is -not the case. - -Next we have Sender: - -[snippet,java] --------------- -source=core/io/src/main/java/org/apache/zest/io/Sender.java -tag=sender -------------- - -The Output invokes sendTo and passes in a Receiver that the Sender will use to send individual items. The sender at -this point can start transferring data of type T to the receiver, one at a time. The Receiver looks like this: - -[snippet,java] --------------- -source=core/io/src/main/java/org/apache/zest/io/Receiver.java -tag=receiver -------------- - -When the receiver gets the individual items from the sender it can either immediately write them to its underlying -resource, or batch them up. Since the receiver will know when the transfer is done (sendTo returns) it can write the -remaining batches properly and close any resource it holds. - -This simple pattern, with two interfaces on the sending side and two on the receiving side, gives us the potential to -do scalable, performant and fault-tolerant transfers of data. - -== Standard Inputs and Outputs == - -So now that the above API defines the contract of sending and receiving data, I can then create a couple of standard -inputs and outputs. Let's say, reading lines of text from a file, and writing lines of text to a file. These -implementations I can then put in static methods so they are easy to use. In the end, to make a copy of a text file -looks like this: - -[snippet,java] --------------- -source=manual/src/main/java/org/apache/zest/manual/recipes/io/Docs.java -tag=filter --------------- - -One line of code that handles the reading, the writing, the cleaning up, buffering, and whatnot. Pretty nifty! The -transferTo method will throw IOException, which I can catch if I want to present any errors to the user. But actually -dealing with those errors, i.e. closing the files and potentially deleting the destination if the transfer failed, is -already handled by the Input and Output. I will never have to deal with the details of reading text from a file ever -again! - -== Intercepting the transfer == - -While the above handles the basic input/output of a transfer, there are usually other things that I want to do as well. -I may want to count how many items were transferred, do some filtering, or log every 1000 items or so to see what's -going on. Since input and output are now separated this becomes simply a matter of inserting something in the middle -that mediates the input and output. Since many of these mediations have a similar character I can put these into -standard utility methods to make them easier to use. - -The first standard decorator is a filter. I will implement this by means of supplying a Specification: - -[source,java] --------------- -public static <T,ReceiverThrowableType extends Throwable> Output<T, ReceiverThrowableType> filter( final Specification<T> specification, final Output<T, ReceiverThrowableType> output) -{ - ... create an Output that filters items based on the Specification<T> ... -} --------------- - -Where Specification is: - -[source,java] --------------- -interface Specification<T> -{ - boolean test(T item); -} --------------- - -With this simple construct I can now perform transfers and easily filter out items I don't want on the receiving side. -This example removes empty lines from a text file. - -[source,java] --------------- -File source = ... -File destination = ... -Inputs.text( source ).transferTo( Transforms.filter(new Specification<String>() -{ - public boolean test(String string) - { - return string.length() != 0; - } -}, Outputs.text(destination) ); --------------- - - -The second common operation is mapping from one type to the other. This deals with the case that one Input you have may -not match the Output you want to send to, but there's a way to map from the input type to the output type. An example -would be to map from String to JSONObject, for example. The operation itself looks like this: - -[source,java] --------------- -public static <From,To,ReceiverThrowableType extends Throwable> Output<From, ReceiverThrowableType> map( Function<From,To> function, Output<To, ReceiverThrowableType> output) --------------- - -Where Function is defined as: - -[source,java] --------------- -interface Function<From, To> -{ - To map(From from); -} --------------- - -With this I can then connect an Input of Strings to an Output of JSONObject like so: - -[source,java] --------------- -Input<String,IOException> input = ...; -Output<JSONObject,RuntimeException> output = ...; -input.transferTo(Transforms.map(new String2JSON(), output); --------------- - -Where String2JSON implements Function and it's map method converts the String into a JSONObject. - -At this point we can now deal with the last part of the initial example, the counting of items. This can be implemented -as a generic Map that has the same input and output type, and just maintains a count internally that updates on every -call to map(). The example can then be written as: - -[source,java] --------------- -File source = ... -File destination = ... -Counter<String> counter = new Counter<String>(); -Inputs.text( source ).transferTo( Transforms.map(counter, Outputs.text(destination) )); -System.out.println("Nr of lines:"+counter.getCount()) --------------- - -== Usage in the Zest⢠SPI == - -Now I can finally get back to my initial problem that led me to look into this: how to implement a good way to access -EntityStates in a Zest⢠EntityStore, and perform restores of backups. The current version of accessing EntityStates look -like this: - -[source,java] --------------- -<ThrowableType extends Throwable> void visitEntityStates( EntityStateVisitor<ThrowableType> visitor, ModuleSPI module ) - throws ThrowableType; - -interface EntityStateVisitor<ThrowableType extends Throwable> -{ - void visitEntityState( EntityState entityState ) - throws ThrowableType; -} --------------- - -This can now be replaced with: - -[source,java] --------------- -Input<EntityState, EntityStoreException> entityStates(ModuleSPI module); --------------- - -Because of the pattern outlined above, users of this will get more information about what's happening in the traversal, -such as if the EntityStore raised an EntityStoreException during the traversal, which they can then handle gracefully. -It also becomes easy to add decorators such as maps and filters to users of this. Let's say you only are interested in -EntityState's of a given type. Then add a filter for this, without changing the consumer. - -For importing backup data into an EntityStore, the interface used to look like this: - -[source,java] --------------- -interface ImportSupport -{ - ImportResult importFrom( Reader in ) - throws IOException; -} --------------- - -This ties the EntityStore to only being able to read JSON lines from Reader's, the client will not know if the -IOException raised is due to errors in the Reader or writing in the store, and the ImportResult, which contains a list -of exceptions and count of stuff, is quite ugly to create and use. With the I/O API at hand this can now be replaced -with: - -[source,java] --------------- -interface ImportSupport -{ - Output<String,EntityStoreException> importJSON(); -} --------------- - - -To use this, given the helpers outlined above, is as simple as: - -[source,java] --------------- -File backup = ... -ImportSupport entityStore = ... -Inputs.text(backup).transferTo(entityStore.importJSON()); --------------- - -If the client wants any "extras", such as counting how many objects were imported, this can be done by adding filters -as previously shown. If you only want to, say, import entities modified before a particular date (let's say you know -some junk was introduced after a given time), then add a specification filter that performs this check. And so on. - -== Conclusion == - -It is quite common while developing software that you have to shuffle data or objects from one input to another output, -possible with some transformations in the middle. Usually these things have to be done from scratch, which opens up for -errors and badly applied patterns. By introducing a generic Input/Output API that encapsulates and separates these -things properly it becomes easier to perform these tasks in a scalable, performant and error-free way, and while still -allowing these tasks to be decorated with extra features when needed. - -This article has outlined one way to do this, and the API and helpers that I've described are available in the current -Zest⢠Core 1.3-SNAPSHOT in Git (see Zest⢠homepage for access details). The idea is to start using it throughout Zest -wherever we need to do I/O of the type described here. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/manual/src/docs/userguide/core.txt ---------------------------------------------------------------------- diff --git a/manual/src/docs/userguide/core.txt b/manual/src/docs/userguide/core.txt index b6d77de..588de3c 100644 --- a/manual/src/docs/userguide/core.txt +++ b/manual/src/docs/userguide/core.txt @@ -75,15 +75,6 @@ such as Spring or Java EE applications. <<core-functional,Learn more>> //____ -//*<<core-io,Core I/O API>>* -=== Core I/O API === -//____ -The Zest⢠Core I/O API tries to address the problem around shuffling data around from various I/O inputs and outputs, -possibly with transformations and filtering along the way. - -<<core-io,Learn more>> -//____ - //*<<core-spi,Core Extension SPI>>* === Core Extension SPI === //____ @@ -123,10 +114,6 @@ include::../../../../core/functional/src/docs/functional.txt[] :leveloffset: 2 -include::../../../../core/io/src/docs/io.txt[] - -:leveloffset: 2 - include::../../../../core/spi/src/docs/spi.txt[] :leveloffset: 2 http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/manual/src/docs/website/tutorials.txt ---------------------------------------------------------------------- diff --git a/manual/src/docs/website/tutorials.txt b/manual/src/docs/website/tutorials.txt index 256d8e8..36f9a04 100644 --- a/manual/src/docs/website/tutorials.txt +++ b/manual/src/docs/website/tutorials.txt @@ -65,7 +65,6 @@ typical application. - <<howto-create-sideeffect>> - <<howto-create-entity>> - <<howto-configure-service>> -- <<howto-use-io>> === Zest⢠Development === @@ -143,10 +142,6 @@ include::../tutorials/howto-invocation-annotation.txt[] :leveloffset: 2 -include::../tutorials/howto-use-io.txt[] - -:leveloffset: 2 - include::../tutorials/howto-build-system.txt[] :leveloffset: 2 http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/manual/src/main/java/org/apache/zest/manual/recipes/io/Docs.java ---------------------------------------------------------------------- diff --git a/manual/src/main/java/org/apache/zest/manual/recipes/io/Docs.java b/manual/src/main/java/org/apache/zest/manual/recipes/io/Docs.java deleted file mode 100644 index 23c0cd6..0000000 --- a/manual/src/main/java/org/apache/zest/manual/recipes/io/Docs.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * - */ -package org.apache.zest.manual.recipes.io; - -import java.io.File; -import java.io.IOException; -import org.apache.zest.io.Inputs; -import org.apache.zest.io.Outputs; - -public class Docs -{ - - public void filter() - throws IOException - { -// START SNIPPET: filter - File source = new File("source.txt"); - File destination = new File("destination.txt"); - Inputs.text( source ).transferTo( Outputs.text( destination ) ); -// END SNIPPET: filter - } -} http://git-wip-us.apache.org/repos/asf/zest-java/blob/37ce367a/settings.gradle ---------------------------------------------------------------------- diff --git a/settings.gradle b/settings.gradle index 0086623..97a817f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,7 +19,6 @@ */ include 'core:functional', - 'core:io', 'core:api', 'core:spi', 'core:testsupport',
