This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/arrow-flight-sql-postgresql.git
The following commit(s) were added to refs/heads/main by this push:
new 29ce3cc [CI] Add a minimal test (#8)
29ce3cc is described below
commit 29ce3cc86b1582f24a34d3ff92f41be715440d6f
Author: Sutou Kouhei <[email protected]>
AuthorDate: Mon Feb 6 16:23:30 2023 +0900
[CI] Add a minimal test (#8)
Close GH-2
---
.github/workflows/test.yaml | 82 ++++++++++++
Gemfile | 24 ++++
test/helper/sandbox.rb | 307 ++++++++++++++++++++++++++++++++++++++++++++
test/run.rb | 24 ++++
test/test-flight-sql.rb | 29 +++++
5 files changed, 466 insertions(+)
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
new file mode 100644
index 0000000..0b8436f
--- /dev/null
+++ b/.github/workflows/test.yaml
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Test
+
+on:
+ pull_request:
+ push:
+
+concurrency:
+ group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ ubuntu:
+ name: Ubuntu - PostgreSQL ${{ matrix.postgresql-version }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ strategy:
+ fail-fast: false
+ matrix:
+ postgresql-version:
+ - "15"
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install Apache Arrow
+ run: |
+ sudo apt update
+ sudo apt install -y -V ca-certificates lsb-release wget
+ apt_source_deb=apache-arrow-apt-source-latest-$(lsb_release
--codename --short).deb
+ wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id
--short | tr 'A-Z' 'a-z')/${apt_source_deb}
+ sudo apt install -y -V ./${apt_source_deb}
+ sudo apt update
+ sudo apt install -y -V \
+ libarrow-flight-sql-glib-dev \
+ ninja-build
+ - name: Install PostgreSQL
+ run: |
+ sudo apt -y -V purge '^postgresql'
+ suite=$(lsb_release --codename --short)-pgdg
+ sudo tee /etc/apt/sources.list.d/pgdg.list <<APT_SOURCE
+ deb http://apt.postgresql.org/pub/repos/apt ${suite} main
+ APT_SOURCE
+ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc
| sudo apt-key add -
+ sudo apt update
+ sudo apt -y -V -t ${suite} install \
+ postgresql-${{ matrix.postgresql-version }} \
+ postgresql-server-dev-${{ matrix.postgresql-version }}
+ - name: Install Apache Arrow Flight SQL adapter
+ run: |
+ cmake \
+ -S . \
+ -B build \
+ -DCMAKE_INSTALL_PREFIX=/usr \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -GNinja
+ ninja -C build
+ sudo ninja -C build install
+ - uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: ruby
+ bundler-cache: true
+ - name: Test
+ run: |
+ PATH=$(pg_config --bindir):$PATH bundle exec test/run.rb
diff --git a/Gemfile b/Gemfile
new file mode 100644
index 0000000..66ab32d
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,24 @@
+# -*- ruby -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source "https://rubygems.org";
+
+gem "rake"
+gem "red-arrow-flight-sql"
+gem "test-unit"
diff --git a/test/helper/sandbox.rb b/test/helper/sandbox.rb
new file mode 100644
index 0000000..90bcf37
--- /dev/null
+++ b/test/helper/sandbox.rb
@@ -0,0 +1,307 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "fileutils"
+require "socket"
+
+require "arrow-flight-sql"
+
+module Helper
+ module CommandRunnable
+ def spawn_process(*args)
+ env = {
+ "LC_ALL" => "C",
+ "PGCLIENTENCODING" => "UTF-8",
+ }
+ output_read, output_write = IO.pipe
+ error_read, error_write = IO.pipe
+ options = {
+ :out => output_write,
+ :err => error_write,
+ }
+ pid = spawn(env, *args, options)
+ output_write.close
+ error_write.close
+ [pid, output_read, error_read]
+ end
+
+ def read_command_output(input)
+ return "" unless IO.select([input], nil, nil, 0)
+ begin
+ data = input.readpartial(4096).gsub(/\r\n/, "\n")
+ data.force_encoding("UTF-8")
+ data
+ rescue EOFError
+ ""
+ end
+ end
+
+ def run_command(*args)
+ pid, output_read, error_read = spawn_process(*args)
+ output = ""
+ error = ""
+ status = nil
+ timeout = 1
+ loop do
+ readables, = IO.select([output_read, error_read], nil, nil, timeout)
+ if readables
+ timeout = 0
+ readables.each do |readable|
+ if readable == output_read
+ output << read_command_output(output_read)
+ else
+ error << read_command_output(error_read)
+ end
+ end
+ else
+ timeout = 1
+ end
+ _, status = Process.waitpid2(pid, Process::WNOHANG)
+ break if status
+ end
+ output << read_command_output(output_read)
+ error << read_command_output(error_read)
+ unless status.success?
+ command_line = args.join(" ")
+ message = "failed to run: #{command_line}\n"
+ message << "output:\n"
+ message << output
+ message << "error:\n"
+ message << error
+ raise message
+ end
+ [output, error]
+ end
+ end
+
+ class PostgreSQL
+ include CommandRunnable
+
+ attr_reader :dir
+ attr_reader :host
+ attr_reader :port
+ attr_reader :flight_sql_port
+ attr_reader :flight_sql_uri
+ attr_reader :user
+ def initialize(base_dir)
+ @base_dir = base_dir
+ @dir = nil
+ @log_base_name = "postgresql.log"
+ @log_path = nil
+ @host = "127.0.0.1"
+ @port = nil
+ @flight_sql_port = nil
+ @flight_sql_uri = nil
+ @user = "arrow-flight-sql-test"
+ @running = false
+ end
+
+ def running?
+ @running
+ end
+
+ def initdb(shared_preload_libraries: [],
+ db_path: "db",
+ port: 25432,
+ flight_sql_port: 35432)
+ @dir = File.join(@base_dir, db_path)
+ @log_path = File.join(@dir, "log", @log_base_name)
+ socket_dir = File.join(@dir, "socket")
+ @port = port
+ @flight_sql_port = flight_sql_port
+ @flight_sql_uri = "grpc://#{@host}:#{@flight_sql_port}"
+ run_command("initdb",
+ "--locale", "C",
+ "--encoding", "UTF-8",
+ "--username", @user,
+ "-D", @dir)
+ FileUtils.mkdir_p(socket_dir)
+ postgresql_conf = File.join(@dir, "postgresql.conf")
+ File.open(postgresql_conf, "a") do |conf|
+ conf.puts("listen_addresses = '#{@host}'")
+ conf.puts("port = #{@port}")
+ unless windows?
+ conf.puts("unix_socket_directories = '#{socket_dir}'")
+ end
+ conf.puts("logging_collector = on")
+ conf.puts("log_filename = '#{@log_base_name}'")
+ conf.puts("shared_preload_libraries = " +
+ "'#{shared_preload_libraries.join(",")}'")
+ conf.puts("arrow_flight_sql.uri = #{@flight_sql_uri}")
+ yield(conf) if block_given?
+ end
+ end
+
+ def start
+ begin
+ run_command("pg_ctl", "start",
+ "-w",
+ "-D", @dir)
+ rescue => error
+ error.message << "\nPostgreSQL log:\n#{read_log}"
+ raise
+ end
+ loop do
+ begin
+ TCPSocket.open(@host, @port) do
+ end
+ rescue SystemCallError
+ sleep(0.1)
+ else
+ break
+ end
+ end
+ @running = true
+ end
+
+ def stop
+ return unless running?
+ run_command("pg_ctl", "stop",
+ "-D", @dir)
+ end
+
+ def psql(db, sql)
+ output, error = run_command("psql",
+ "--host", @host,
+ "--port", @port.to_s,
+ "--username", @user,
+ "--dbname", db,
+ "--echo-all",
+ "--no-psqlrc",
+ "--command", sql)
+ [output, error]
+ end
+
+ def flight_sql_client
+ client = ArrowFlight::Client.new(@flight_sql_uri)
+ ArrowFlightSQL::Client.new(client)
+ end
+
+ def read_log
+ return "" unless File.exist?(@log_path)
+ File.read(@log_path)
+ end
+
+ private
+ def windows?
+ /mingw|mswin|cygwin/.match?(RUBY_PLATFORM)
+ end
+ end
+
+ module Sandbox
+ include CommandRunnable
+
+ class << self
+ def included(base)
+ base.module_eval do
+ setup :setup_tmp_dir
+ teardown :teardown_tmp_dir
+
+ setup :setup_db
+ teardown :teardown_db
+
+ setup :setup_postgres
+ teardown :teardown_postgres
+
+ setup :setup_test_db
+ teardown :teardown_test_db
+ end
+ end
+ end
+
+ def psql(db, sql)
+ @postgresql.psql(db, sql)
+ end
+
+ def run_sql(sql)
+ psql(@test_db_name, sql)
+ end
+
+ def flight_sql_client
+ @postgresql.flight_sql_client
+ end
+
+ def setup_tmp_dir
+ memory_fs = "/dev/shm"
+ if File.exist?(memory_fs)
+ @tmp_dir = File.join(memory_fs, "arrow-flight-sql")
+ else
+ @tmp_dir = File.join(__dir__, "tmp")
+ end
+ FileUtils.rm_rf(@tmp_dir)
+ FileUtils.mkdir_p(@tmp_dir)
+ end
+
+ def teardown_tmp_dir
+ debug_dir = ENV["AFS_TEST_DEBUG_DIR"]
+ if debug_dir and File.exist?(@tmp_dir)
+ FileUtils.rm_rf(debug_dir)
+ FileUtils.mv(@tmp_dir, debug_dir)
+ else
+ FileUtils.rm_rf(@tmp_dir)
+ end
+ end
+
+ def setup_db
+ @postgresql = PostgreSQL.new(@tmp_dir)
+ options = {
+ shared_preload_libraries: shared_preload_libraries,
+ }
+ @postgresql.initdb(**options)
+ end
+
+ def shared_preload_libraries
+ ["arrow_flight_sql"]
+ end
+
+ def teardown_db
+ end
+
+ def start_postgres
+ @postgresql.start
+ end
+
+ def stop_postgres
+ @postgresql.stop
+ end
+
+ def setup_postgres
+ start_postgres
+ end
+
+ def teardown_postgres
+ stop_postgres if @postgresql
+ end
+
+ def create_db(postgresql, db_name)
+ postgresql.psql("postgres", "CREATE DATABASE #{db_name}")
+ postgresql.psql(db_name, "CHECKPOINT")
+ end
+
+ def setup_test_db
+ @test_db_name = "test"
+ create_db(@postgresql, @test_db_name)
+ result, = run_sql("SELECT oid FROM pg_catalog.pg_database " +
+ "WHERE datname = current_database()")
+ oid = result.lines[3].strip
+ @test_db_dir = File.join(@postgresql.dir, "base", oid)
+ end
+
+ def teardown_test_db
+ end
+ end
+end
diff --git a/test/run.rb b/test/run.rb
new file mode 100755
index 0000000..2a5d8a6
--- /dev/null
+++ b/test/run.rb
@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "test-unit"
+
+require_relative "helper/sandbox"
+
+exit(Test::Unit::AutoRunner.run(true, __dir__))
diff --git a/test/test-flight-sql.rb b/test/test-flight-sql.rb
new file mode 100644
index 0000000..e74723b
--- /dev/null
+++ b/test/test-flight-sql.rb
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class FlightSQLTest < Test::Unit::TestCase
+ include Helper::Sandbox
+
+ def test_connect
+ exception = assert_raise(Arrow::Error::NotImplemented) do
+ flight_sql_client.execute("SELECT 1")
+ end
+ assert_equal("[flight-sql-client][execute]: " +
+ "NotImplemented: GetFlightInfoStatement not implemented",
+ exception.message.lines(chomp: true).first)
+ end
+end
