Repository: arrow Updated Branches: refs/heads/master dcaa8e5d7 -> 13c12c6ea
ARROW-682: [Integration] Check implementations against themselves This adds an additional layer of internal consistency checks Author: Wes McKinney <[email protected]> Closes #433 from wesm/ARROW-682 and squashes the following commits: b33ac7a [Wes McKinney] Run integration tests with same implementation producing and consuming to validate internal consistency Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/13c12c6e Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/13c12c6e Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/13c12c6e Branch: refs/heads/master Commit: 13c12c6ea5e23928268b5c2c7b962d223cca7bd4 Parents: dcaa8e5 Author: Wes McKinney <[email protected]> Authored: Fri Mar 24 11:54:18 2017 +0100 Committer: Uwe L. Korn <[email protected]> Committed: Fri Mar 24 11:54:18 2017 +0100 ---------------------------------------------------------------------- integration/integration_test.py | 56 +++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/13c12c6e/integration/integration_test.py ---------------------------------------------------------------------- diff --git a/integration/integration_test.py b/integration/integration_test.py index 5cd63c5..ec2a38d 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -34,10 +34,12 @@ ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0] # Control for flakiness np.random.seed(12345) + def load_version_from_pom(): import xml.etree.ElementTree as ET tree = ET.parse(os.path.join(ARROW_HOME, 'java', 'pom.xml')) - version_tag = list(tree.getroot().findall('{http://maven.apache.org/POM/4.0.0}version'))[0] + tag_pattern = '{http://maven.apache.org/POM/4.0.0}version' + version_tag = list(tree.getroot().findall(tag_pattern))[0] return version_tag.text @@ -596,32 +598,32 @@ class IntegrationRunner(object): def run(self): for producer, consumer in itertools.product(self.testers, self.testers): - if producer is consumer: - continue - - print('-- {0} producing, {1} consuming'.format(producer.name, - consumer.name)) - - for json_path in self.json_files: - print('Testing file {0}'.format(json_path)) - - # Make the random access file - print('-- Creating binary inputs') - producer_file_path = os.path.join(self.temp_dir, guid()) - producer.json_to_file(json_path, producer_file_path) - - # Validate the file - print('-- Validating file') - consumer.validate(json_path, producer_file_path) - - print('-- Validating stream') - producer_stream_path = os.path.join(self.temp_dir, guid()) - consumer_file_path = os.path.join(self.temp_dir, guid()) - producer.file_to_stream(producer_file_path, - producer_stream_path) - consumer.stream_to_file(producer_stream_path, - consumer_file_path) - consumer.validate(json_path, consumer_file_path) + self._compare_implementations(producer, consumer) + + def _compare_implementations(self, producer, consumer): + print('-- {0} producing, {1} consuming'.format(producer.name, + consumer.name)) + + for json_path in self.json_files: + print('Testing file {0}'.format(json_path)) + + # Make the random access file + print('-- Creating binary inputs') + producer_file_path = os.path.join(self.temp_dir, guid()) + producer.json_to_file(json_path, producer_file_path) + + # Validate the file + print('-- Validating file') + consumer.validate(json_path, producer_file_path) + + print('-- Validating stream') + producer_stream_path = os.path.join(self.temp_dir, guid()) + consumer_file_path = os.path.join(self.temp_dir, guid()) + producer.file_to_stream(producer_file_path, + producer_stream_path) + consumer.stream_to_file(producer_stream_path, + consumer_file_path) + consumer.validate(json_path, consumer_file_path) class Tester(object):
