Author: hammer
Date: Thu Jun 10 04:33:48 2010
New Revision: 953214
URL: http://svn.apache.org/viewvc?rev=953214&view=rev
Log:
AVRO-331. Inline shared state during the Python build process
(version, handshake schemas, and interop data directory)
(Patrick Wendell via hammer)
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/py/build.xml
avro/trunk/lang/py/setup.py
avro/trunk/lang/py/src/avro/ipc.py
avro/trunk/lang/py/test/test_datafile_interop.py
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=953214&r1=953213&r2=953214&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jun 10 04:33:48 2010
@@ -31,6 +31,10 @@ Avro 1.4.0 (unreleased)
AVRO-284. Handle namespaces correctly in new Python implementation
(Patrick Wendell via hammer)
+ AVRO-331. Inline shared state during the Python build process
+ (version, handshake schemas, and interop data directory)
+ (Patrick Wendell via hammer)
+
BUG FIXES
AVRO-502. Memory leak from parsing JSON schema.
Modified: avro/trunk/lang/py/build.xml
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/py/build.xml?rev=953214&r1=953213&r2=953214&view=diff
==============================================================================
--- avro/trunk/lang/py/build.xml (original)
+++ avro/trunk/lang/py/build.xml Thu Jun 10 04:33:48 2010
@@ -19,15 +19,25 @@
<project name="Avro" default="dist">
<!-- Load user's default properties. -->
- <property file="${user.home}/build.properties" />
+ <property file="${user.home}/build.properties"/>
+ <!-- Shared directories -->
<property name="share.dir" value="${basedir}/../../share"/>
+ <property name="share.schema.dir" value="${share.dir}/schemas/"/>
<property name="dist.dir" value="${basedir}/../../dist/py"/>
<property name="top.build" value="${basedir}/../../build"/>
+ <property name="interop.data.dir" value="${top.build}/interop/data"/>
- <loadresource property="version">
- <file file="${share.dir}/VERSION.txt"/>
- </loadresource>
+ <!-- Python implementation directories -->
+ <property name="build.dir" value="${basedir}/build"/>
+ <property name="src.dir" value="${basedir}/src"/>
+ <property name="lib.dir" value="${basedir}/lib"/>
+ <property name="test.dir" value="${basedir}/test"/>
+
+ <!-- Load shared properties -->
+ <loadfile srcFile="${share.dir}/VERSION.txt" property="avro.version" />
+ <loadfile
srcFile="${share.schema.dir}/org/apache/avro/ipc/HandshakeRequest.avsc"
property="handshake.request.json"/>
+ <loadfile
srcFile="${share.schema.dir}/org/apache/avro/ipc/HandshakeResponse.avsc"
property="handshake.response.json"/>
<path id="java.classpath">
<fileset dir="lib">
@@ -36,64 +46,122 @@
</path>
<path id="test.path">
- <pathelement location="${basedir}/src"/>
- <pathelement location="${basedir}/test"/>
- <pathelement location="${basedir}/lib"/>
+ <pathelement location="${build.dir}/src"/>
+ <pathelement location="${build.dir}/test"/>
+ <pathelement location="${build.dir}/lib"/>
</path>
- <target name="init">
- <copy todir="${basedir}/src/avro">
- <fileset dir="${share.dir}/schemas/org/apache/avro/ipc">
- <include name="**/*.avsc"/>
- </fileset>
+ <target name="init" description="Create the build directory.">
+ <mkdir dir="${build.dir}"/>
+ </target>
+
+ <target name="build"
+ description="Copy project files to build/ and do string replacement."
+ depends="init">
+ <!-- Copy src/, test/, lib/ -->
+ <copy todir="${build.dir}/src">
+ <fileset dir="${src.dir}">
+ <exclude name="**/*.pyc"/>
+ <exclude name="**/*.py~"/>
+ </fileset>
+ </copy>
+ <copy todir="${build.dir}/test">
+ <fileset dir="${test.dir}">
+ <exclude name="**/*.pyc"/>
+ <exclude name="**/*.py~"/>
+ </fileset>
+ </copy>
+ <copy todir="${build.dir}/lib">
+ <fileset dir="${lib.dir}" />
+ </copy>
+
+ <!-- Inline the handshake schemas -->
+ <copy file="${src.dir}/avro/ipc.py"
+ toFile="${build.dir}/src/avro/ipc.py"
+ overwrite="true">
+ <filterset>
+ <filter token="HANDSHAKE_REQUEST_SCHEMA"
+ value="${handshake.request.json}"/>
+ <filter token="HANDSHAKE_RESPONSE_SCHEMA"
+ value="${handshake.response.json}"/>
+ </filterset>
+ </copy>
+
+ <!-- Inline the Avro version -->
+ <copy file="${basedir}/setup.py"
+ toFile="${build.dir}/setup.py"
+ overwrite="true">
+ <filterset>
+ <filter token="AVRO_VERSION" value="${avro.version}"/>
+ </filterset>
+ </copy>
+
+ <!-- Inline the interop data directory -->
+ <copy file="${test.dir}/test_datafile_interop.py"
+ toFile="${build.dir}/test/test_datafile_interop.py"
+ overwrite="true">
+ <filterset>
+ <filter token="INTEROP_DATA_DIR" value="${interop.data.dir}"/>
+ </filterset>
</copy>
</target>
- <target name="test" depends="init" description="Run python unit tests">
+ <target name="test"
+ description="Run python unit tests"
+ depends="build">
<taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
classpathref="java.classpath"/>
<py-test python="python" pythonpathref="test.path" >
- <fileset dir="${basedir}/test">
+ <fileset dir="${build.dir}/test">
<include name="test_*.py"/>
<exclude name="test_datafile_interop.py"/>
</fileset>
</py-test>
+ <delete dir="${build.dir}"/>
</target>
- <target name="interop-data-test" description="Run python interop data tests">
+ <target name="interop-data-test"
+ description="Run python interop data tests"
+ depends="build">
<taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
classpathref="java.classpath"/>
<py-test python="python" pythonpathref="test.path" >
- <fileset dir="${basedir}/test">
+ <fileset dir="${build.dir}/test">
<include name="test_datafile_interop.py"/>
</fileset>
</py-test>
+ <delete dir="${build.dir}"/>
</target>
<target name="interop-data-generate"
- description="Generate Python interop data files.">
- <mkdir dir="${top.build}/interop/data"/>
+ description="Generate Python interop data files."
+ depends="build">
+ <mkdir dir="${interop.data.dir}"/>
<exec executable="python">
- <env key="PYTHONPATH" value="$PYTHONPATH:${basedir}/src"/>
- <arg value="${basedir}/test/gen_interop_data.py"/>
+ <env key="PYTHONPATH" value="$PYTHONPATH:${build.dir}/src"/>
+ <arg value="${build.dir}/test/gen_interop_data.py"/>
<arg value="${share.dir}/test/schemas/interop.avsc"/>
- <arg value="${top.build}/interop/data/py.avro"/>
+ <arg value="${interop.data.dir}/py.avro"/>
</exec>
+ <delete dir="${build.dir}"/>
</target>
- <target name="dist" description="Build egg">
+ <target name="dist"
+ description="Build source distribution"
+ depends="build">
<mkdir dir="${dist.dir}"/>
<exec executable="python" failonerror="true">
- <arg value="setup.py"/>
- <arg value="bdist_egg"/>
+ <arg value="${build.dir}/setup.py"/>
+ <arg value="sdist"/>
<arg value="--dist-dir=${dist.dir}"/>
</exec>
</target>
- <target name="clean" description="Delete build files, and their directories">
- <delete>
- <fileset dir="src" includes="**/*.pyc" />
- <fileset dir="${basedir}/src/avro" includes="**/*.avsc"/>
+ <target name="clean"
+ description="Delete build files and their directories">
+ <delete includeemptydirs="true" failonerror="false">
+ <fileset file="MANIFEST"/>
+ <fileset dir="${build.dir}"/>
</delete>
</target>
Modified: avro/trunk/lang/py/setup.py
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/py/setup.py?rev=953214&r1=953213&r2=953214&view=diff
==============================================================================
--- avro/trunk/lang/py/setup.py (original)
+++ avro/trunk/lang/py/setup.py Thu Jun 10 04:33:48 2010
@@ -20,11 +20,9 @@ try:
except ImportError:
from distutils.core import setup
-VERSION_FILE='../../share/VERSION.txt'
-
setup(
name = 'avro',
- version = file(VERSION_FILE, 'r').read(),
+ version = '@AVRO_VERSION@',
packages = ['avro',],
package_dir = {'avro': 'src/avro'},
Modified: avro/trunk/lang/py/src/avro/ipc.py
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/ipc.py?rev=953214&r1=953213&r2=953214&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/ipc.py (original)
+++ avro/trunk/lang/py/src/avro/ipc.py Thu Jun 10 04:33:48 2010
@@ -30,34 +30,13 @@ from avro import schema
# Constants
#
-HANDSHAKE_REQUEST_SCHEMA = schema.parse("""\
-{
- "type": "record",
- "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
- "fields": [
- {"name": "clientHash",
- "type": {"type": "fixed", "name": "MD5", "size": 16}},
- {"name": "clientProtocol", "type": ["null", "string"]},
- {"name": "serverHash", "type": "MD5"},
- {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
- ]
-}""")
-
-HANDSHAKE_RESPONSE_SCHEMA = schema.parse("""\
-{
- "type": "record",
- "name": "HandshakeResponse", "namespace": "org.apache.avro.ipc",
- "fields": [
- {"name": "match",
- "type": {"type": "enum", "name": "HandshakeMatch",
- "symbols": ["BOTH", "CLIENT", "NONE"]}},
- {"name": "serverProtocol", "type": ["null", "string"]},
- {"name": "serverHash",
- "type": ["null", {"type": "fixed", "name": "MD5", "size": 16}]},
- {"name": "meta",
- "type": ["null", {"type": "map", "values": "bytes"}]}
- ]
-}
+# Handshake schema is pulled in during build
+HANDSHAKE_REQUEST_SCHEMA = schema.parse("""
+...@handshake_request_schema@
+""")
+
+HANDSHAKE_RESPONSE_SCHEMA = schema.parse("""
+...@handshake_response_schema@
""")
HANDSHAKE_REQUESTOR_WRITER = io.DatumWriter(HANDSHAKE_REQUEST_SCHEMA)
Modified: avro/trunk/lang/py/test/test_datafile_interop.py
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_datafile_interop.py?rev=953214&r1=953213&r2=953214&view=diff
==============================================================================
--- avro/trunk/lang/py/test/test_datafile_interop.py (original)
+++ avro/trunk/lang/py/test/test_datafile_interop.py Thu Jun 10 04:33:48 2010
@@ -18,20 +18,18 @@ import unittest
from avro import io
from avro import datafile
-INTEROP_DATA_DIR = os.path.join(os.path.dirname(__file__),
'../../../build/interop/data')
-
class TestDataFileInterop(unittest.TestCase):
def test_interop(self):
print ''
print 'TEST INTEROP'
print '============'
print ''
- for f in os.listdir(INTEROP_DATA_DIR):
+ for f in os.listdir('@INTEROP_DATA_DIR@'):
print 'READING %s' % f
print ''
# read data in binary from file
- reader = open(os.path.join(INTEROP_DATA_DIR, f), 'rb')
+ reader = open(os.path.join('@INTEROP_DATA_DIR@', f), 'rb')
datum_reader = io.DatumReader()
dfr = datafile.DataFileReader(reader, datum_reader)
for datum in dfr:
