This is an automated email from the ASF dual-hosted git repository.
ianmcook pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-experiments.git
The following commit(s) were added to refs/heads/main by this push:
new 188c4e5 Add Julia example [WIP] (#29)
188c4e5 is described below
commit 188c4e5ff4bda08319d4520e380d736c36b9ee48
Author: Simone Carlo Surace <[email protected]>
AuthorDate: Wed May 1 15:32:57 2024 +0200
Add Julia example [WIP] (#29)
* Add Julia example
* Increase number of records
* Make server blocking
* Make columns nullable
---
http/get_simple/julia/Project.toml | 5 ++++
http/get_simple/julia/client/README.md | 32 +++++++++++++++++++++++++
http/get_simple/julia/client/client.jl | 33 +++++++++++++++++++++++++
http/get_simple/julia/server/README.md | 32 +++++++++++++++++++++++++
http/get_simple/julia/server/server.jl | 44 ++++++++++++++++++++++++++++++++++
5 files changed, 146 insertions(+)
diff --git a/http/get_simple/julia/Project.toml
b/http/get_simple/julia/Project.toml
new file mode 100644
index 0000000..0929efe
--- /dev/null
+++ b/http/get_simple/julia/Project.toml
@@ -0,0 +1,5 @@
+[deps]
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
diff --git a/http/get_simple/julia/client/README.md
b/http/get_simple/julia/client/README.md
new file mode 100644
index 0000000..3dbed30
--- /dev/null
+++ b/http/get_simple/julia/client/README.md
@@ -0,0 +1,32 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# HTTP GET Arrow Data: Simple Julia Client Example
+
+This directory contains a minimal example of an HTTP client implemented in
Julia. The client:
+1. Sends an HTTP GET request to a server.
+2. Receives an HTTP 200 response from the server, with the response body
containing an Arrow IPC stream of record batches.
+3. Adds the record batches to a list as they are received.
+
+To run this example, first start one of the server examples in the parent
directory, then:
+
+```sh
+julia --project=.. -e "using Pkg; Pkg.instantiate()"
+julia --project=.. client.jl
+```
diff --git a/http/get_simple/julia/client/client.jl
b/http/get_simple/julia/client/client.jl
new file mode 100644
index 0000000..1aa2073
--- /dev/null
+++ b/http/get_simple/julia/client/client.jl
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+using Arrow, HTTP
+
+function get_batches()
+ res = HTTP.get("http://localhost:8008";)
+ buffer = res.body
+ stream = Arrow.Stream(res.body)
+ batches = collect(stream)
+
+ println("$(length(buffer)) bytes received")
+ println("$(length(batches)) record batches received")
+
+ return batches
+end
+
+execution_time = @elapsed get_batches()
+println("$(execution_time) seconds elapsed")
diff --git a/http/get_simple/julia/server/README.md
b/http/get_simple/julia/server/README.md
new file mode 100644
index 0000000..19abc3a
--- /dev/null
+++ b/http/get_simple/julia/server/README.md
@@ -0,0 +1,32 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# HTTP GET Arrow Data: Simple Julia Server Example
+
+This directory contains a minimal example of an HTTP server implemented in
Julia. The server:
+1. Creates a list of record batches and populates it with synthesized data.
+2. Listens for HTTP GET requests from clients.
+3. Upon receiving a request, sends an HTTP 200 response with the body
containing an Arrow IPC stream of record batches.
+
+To run this example:
+
+```sh
+julia --project=.. -e "using Pkg; Pkg.instantiate()"
+julia --project=.. server.jl
+```
diff --git a/http/get_simple/julia/server/server.jl
b/http/get_simple/julia/server/server.jl
new file mode 100644
index 0000000..8f56524
--- /dev/null
+++ b/http/get_simple/julia/server/server.jl
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+using Arrow, HTTP, Random, Tables
+
+function randint_nullable(n::Integer)
+ v = Vector{Union{Missing, Int}}(undef, n)
+ rand!(v, Int)
+ return v
+end
+
+function get_stream(::HTTP.Request)
+ total_records = 100_000_000
+ batch_len = 4096
+ stream = Tables.partitioner(Iterators.partition(1:total_records,
batch_len)) do indices
+ nrows = length(indices)
+ return (
+ a = randint_nullable(nrows),
+ b = randint_nullable(nrows),
+ c = randint_nullable(nrows),
+ d = randint_nullable(nrows)
+ )
+ end
+ buffer = IOBuffer()
+ Arrow.write(buffer, stream)
+ return HTTP.Response(200, take!(buffer))
+end
+
+println("Serving on localhost:8008...")
+server = HTTP.serve(get_stream, "127.0.0.1", 8008)
