Author: dcreager
Date: Fri May 24 13:49:20 2013
New Revision: 1486053
URL: http://svn.apache.org/r1486053
Log:
AVRO-1324. C: Handle namespaces when parsing schemas
The schema parser can now handle fully-qualified schema references, and it
understands inherited namespaces.
Contributed by Ben Walsh.
Added:
avro/trunk/lang/c/tests/schema_tests/pass/namespace_recursive
avro/trunk/lang/c/tests/schema_tests/pass/namespace_simple
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c/src/allocation.c
avro/trunk/lang/c/src/avro/allocation.h
avro/trunk/lang/c/src/avro_private.h
avro/trunk/lang/c/src/schema.c
avro/trunk/lang/c/src/schema_equal.c
avro/trunk/lang/c/tests/test_avro_schema.c
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri May 24 13:49:20 2013
@@ -59,6 +59,9 @@ Trunk (not yet released)
AVRO-1238. C: EOF detection in avro_file_reader_read_value.
(Michael Cooper via dcreager)
+ AVRO-1324. C: Handle namespaces in schema parsing.
+ (Ben Walsh via dcreager)
+
BUG FIXES
AVRO-1296. Python: Fix schemas retrieved from protocol types
Modified: avro/trunk/lang/c/src/allocation.c
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/allocation.c?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/src/allocation.c (original)
+++ avro/trunk/lang/c/src/allocation.c Fri May 24 13:49:20 2013
@@ -57,13 +57,8 @@ void *avro_calloc(size_t count, size_t s
return ptr;
}
-char *avro_strdup(const char *str)
+char *avro_str_alloc(size_t str_size)
{
- if (str == NULL) {
- return NULL;
- }
-
- size_t str_size = strlen(str)+1;
size_t buf_size = str_size + sizeof(size_t);
void *buf = avro_malloc(buf_size);
@@ -75,6 +70,18 @@ char *avro_strdup(const char *str)
char *new_str = (char *) (size + 1);
*size = buf_size;
+
+ return new_str;
+}
+
+char *avro_strdup(const char *str)
+{
+ if (str == NULL) {
+ return NULL;
+ }
+
+ size_t str_size = strlen(str)+1;
+ char *new_str = avro_str_alloc(str_size);
memcpy(new_str, str, str_size);
//fprintf(stderr, "--- new %" PRIsz " %p %s\n", *size, new_str,
new_str);
Modified: avro/trunk/lang/c/src/avro/allocation.h
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/avro/allocation.h?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/src/avro/allocation.h (original)
+++ avro/trunk/lang/c/src/avro/allocation.h Fri May 24 13:49:20 2013
@@ -80,6 +80,7 @@ void *avro_calloc(size_t count, size_t s
* avro_str_free is a string created via avro_strdup.
*/
+char *avro_str_alloc(size_t str_size);
char *avro_strdup(const char *str);
void avro_str_free(char *str);
Modified: avro/trunk/lang/c/src/avro_private.h
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/avro_private.h?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/src/avro_private.h (original)
+++ avro/trunk/lang/c/src/avro_private.h Fri May 24 13:49:20 2013
@@ -92,5 +92,8 @@ extern "C" {
#define container_of(ptr_, type_, member_) \
((type_ *)((char *)ptr_ - (size_t)&((type_ *)0)->member_))
+#define nullstrcmp(s1, s2) \
+ (((s1) && (s2)) ? strcmp(s1, s2) : ((s1) || (s2)))
+
CLOSE_EXTERN
#endif
Modified: avro/trunk/lang/c/src/schema.c
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/schema.c?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/src/schema.c (original)
+++ avro/trunk/lang/c/src/schema.c Fri May 24 13:49:20 2013
@@ -34,6 +34,11 @@
#define DEFAULT_TABLE_SIZE 32
+/* forward declaration */
+static int
+avro_schema_to_json2(const avro_schema_t schema, avro_writer_t out,
+ const char *parent_namespace);
+
static void avro_schema_init(avro_schema_t schema, avro_type_t type)
{
schema->type = type;
@@ -682,20 +687,38 @@ avro_schema_t avro_schema_link_target(av
return link->to;
}
+static const char *
+qualify_name(const char *name, const char *namespace)
+{
+ char *full_name;
+ if (namespace != NULL && strchr(name, '.') == NULL) {
+ full_name = avro_str_alloc(strlen(name) + strlen(namespace) +
2);
+ sprintf(full_name, "%s.%s", namespace, name);
+ } else {
+ full_name = avro_strdup(name);
+ }
+ return full_name;
+}
+
static int
-save_named_schemas(const char *name, avro_schema_t schema, st_table *st)
+save_named_schemas(const char *name, const char *namespace, avro_schema_t
schema, st_table *st)
{
- return st_insert(st, (st_data_t) name, (st_data_t) schema);
+ const char *full_name = qualify_name(name, namespace);
+ int rval = st_insert(st, (st_data_t) full_name, (st_data_t) schema);
+ return rval;
}
static avro_schema_t
-find_named_schemas(const char *name, st_table *st)
+find_named_schemas(const char *name, const char *namespace, st_table *st)
{
union {
avro_schema_t schema;
st_data_t data;
} val;
- if (st_lookup(st, (st_data_t) name, &(val.data))) {
+ const char *full_name = qualify_name(name, namespace);
+ int rval = st_lookup(st, (st_data_t) full_name, &(val.data));
+ avro_str_free((char *)full_name);
+ if (rval) {
return val.schema;
}
avro_set_error("No schema type named %s", name);
@@ -704,7 +727,8 @@ find_named_schemas(const char *name, st_
static int
avro_type_from_json_t(json_t *json, avro_type_t *type,
- st_table *named_schemas, avro_schema_t *named_type)
+ st_table *named_schemas, avro_schema_t *named_type,
+ const char *namespace)
{
json_t *json_type;
const char *type_str;
@@ -755,7 +779,7 @@ avro_type_from_json_t(json_t *json, avro
*type = AVRO_MAP;
} else if (strcmp(type_str, "fixed") == 0) {
*type = AVRO_FIXED;
- } else if ((*named_type = find_named_schemas(type_str, named_schemas)))
{
+ } else if ((*named_type = find_named_schemas(type_str, namespace,
named_schemas))) {
*type = AVRO_LINK;
} else {
avro_set_error("Unknown Avro \"type\": %s", type_str);
@@ -766,7 +790,7 @@ avro_type_from_json_t(json_t *json, avro
static int
avro_schema_from_json_t(json_t *json, avro_schema_t *schema,
- st_table *named_schemas)
+ st_table *named_schemas, const char *parent_namespace)
{
#ifdef _WIN32
#pragma message("#warning: Bug: '0' is not of type avro_type_t.")
@@ -780,7 +804,7 @@ avro_schema_from_json_t(json_t *json, av
unsigned int i;
avro_schema_t named_type = NULL;
- if (avro_type_from_json_t(json, &type, named_schemas, &named_type)) {
+ if (avro_type_from_json_t(json, &type, named_schemas, &named_type,
parent_namespace)) {
return EINVAL;
}
@@ -853,11 +877,11 @@ avro_schema_from_json_t(json_t *json, av
record_namespace =
json_string_value(json_namespace);
} else {
- record_namespace = NULL;
+ record_namespace = parent_namespace;
}
*schema =
avro_schema_record(record_name, record_namespace);
- if (save_named_schemas(record_name, *schema,
named_schemas)) {
+ if (save_named_schemas(record_name, record_namespace,
*schema, named_schemas)) {
avro_set_error("Cannot save record schema");
return ENOMEM;
}
@@ -891,7 +915,7 @@ avro_schema_from_json_t(json_t *json, av
field_rval =
avro_schema_from_json_t(json_field_type,
&json_field_type_schema,
- named_schemas);
+ named_schemas,
record_namespace);
if (field_rval) {
avro_schema_decref(*schema);
return field_rval;
@@ -937,7 +961,7 @@ avro_schema_from_json_t(json_t *json, av
return EINVAL;
}
*schema = avro_schema_enum(name);
- if (save_named_schemas(name, *schema, named_schemas)) {
+ if (save_named_schemas(name, parent_namespace, *schema,
named_schemas)) {
avro_set_error("Cannot save enum schema");
return ENOMEM;
}
@@ -974,7 +998,7 @@ avro_schema_from_json_t(json_t *json, av
}
items_rval =
avro_schema_from_json_t(json_items, &items_schema,
- named_schemas);
+ named_schemas,
parent_namespace);
if (items_rval) {
return items_rval;
}
@@ -995,7 +1019,7 @@ avro_schema_from_json_t(json_t *json, av
}
values_rval =
avro_schema_from_json_t(json_values, &values_schema,
- named_schemas);
+ named_schemas,
parent_namespace);
if (values_rval) {
return values_rval;
}
@@ -1022,7 +1046,7 @@ avro_schema_from_json_t(json_t *json, av
}
schema_rval =
avro_schema_from_json_t(schema_json, &s,
- named_schemas);
+ named_schemas,
parent_namespace);
if (schema_rval != 0) {
avro_schema_decref(*schema);
return schema_rval;
@@ -1055,7 +1079,7 @@ avro_schema_from_json_t(json_t *json, av
size = json_integer_value(json_size);
name = json_string_value(json_name);
*schema = avro_schema_fixed(name, (int64_t) size);
- if (save_named_schemas(name, *schema, named_schemas)) {
+ if (save_named_schemas(name, parent_namespace, *schema,
named_schemas)) {
avro_set_error("Cannot save fixed schema");
return ENOMEM;
}
@@ -1069,6 +1093,15 @@ avro_schema_from_json_t(json_t *json, av
return 0;
}
+static int named_schema_free_foreach(char *full_name, st_data_t value,
st_data_t arg)
+{
+ AVRO_UNUSED(value);
+ AVRO_UNUSED(arg);
+
+ avro_str_free(full_name);
+ return ST_DELETE;
+}
+
static int
avro_schema_from_json_root(json_t *root, avro_schema_t *schema)
{
@@ -1083,8 +1116,9 @@ avro_schema_from_json_root(json_t *root,
}
/* json_dumpf(root, stderr, 0); */
- rval = avro_schema_from_json_t(root, schema, named_schemas);
+ rval = avro_schema_from_json_t(root, schema, named_schemas, NULL);
json_decref(root);
+ st_foreach(named_schemas, HASH_FUNCTION_CAST named_schema_free_foreach,
0);
st_free_table(named_schemas);
return rval;
}
@@ -1475,17 +1509,19 @@ static int avro_write_str(avro_writer_t
return avro_write(out, (char *)str, strlen(str));
}
-static int write_field(avro_writer_t out, const struct avro_record_field_t
*field)
+static int write_field(avro_writer_t out, const struct avro_record_field_t
*field,
+ const char *parent_namespace)
{
int rval;
check(rval, avro_write_str(out, "{\"name\":\""));
check(rval, avro_write_str(out, field->name));
check(rval, avro_write_str(out, "\",\"type\":"));
- check(rval, avro_schema_to_json(field->type, out));
+ check(rval, avro_schema_to_json2(field->type, out, parent_namespace));
return avro_write_str(out, "}");
}
-static int write_record(avro_writer_t out, const struct avro_record_schema_t
*record)
+static int write_record(avro_writer_t out, const struct avro_record_schema_t
*record,
+ const char *parent_namespace)
{
int rval;
long i;
@@ -1493,7 +1529,7 @@ static int write_record(avro_writer_t ou
check(rval, avro_write_str(out, "{\"type\":\"record\",\"name\":\""));
check(rval, avro_write_str(out, record->name));
check(rval, avro_write_str(out, "\","));
- if (record->space) {
+ if (nullstrcmp(record->space, parent_namespace)) {
check(rval, avro_write_str(out, "\"namespace\":\""));
check(rval, avro_write_str(out, record->space));
check(rval, avro_write_str(out, "\","));
@@ -1508,7 +1544,7 @@ static int write_record(avro_writer_t ou
if (i) {
check(rval, avro_write_str(out, ","));
}
- check(rval, write_field(out, val.field));
+ check(rval, write_field(out, val.field, record->space));
}
return avro_write_str(out, "]}");
}
@@ -1547,21 +1583,24 @@ static int write_fixed(avro_writer_t out
check(rval, avro_write_str(out, size));
return avro_write_str(out, "}");
}
-static int write_map(avro_writer_t out, const struct avro_map_schema_t *map)
+static int write_map(avro_writer_t out, const struct avro_map_schema_t *map,
+ const char *parent_namespace)
{
int rval;
check(rval, avro_write_str(out, "{\"type\":\"map\",\"values\":"));
- check(rval, avro_schema_to_json(map->values, out));
+ check(rval, avro_schema_to_json2(map->values, out, parent_namespace));
return avro_write_str(out, "}");
}
-static int write_array(avro_writer_t out, const struct avro_array_schema_t
*array)
+static int write_array(avro_writer_t out, const struct avro_array_schema_t
*array,
+ const char *parent_namespace)
{
int rval;
check(rval, avro_write_str(out, "{\"type\":\"array\",\"items\":"));
- check(rval, avro_schema_to_json(array->items, out));
+ check(rval, avro_schema_to_json2(array->items, out, parent_namespace));
return avro_write_str(out, "}");
}
-static int write_union(avro_writer_t out, const struct avro_union_schema_t
*unionp)
+static int write_union(avro_writer_t out, const struct avro_union_schema_t
*unionp,
+ const char *parent_namespace)
{
int rval;
long i;
@@ -1576,19 +1615,29 @@ static int write_union(avro_writer_t out
if (i) {
check(rval, avro_write_str(out, ","));
}
- check(rval, avro_schema_to_json(val.schema, out));
+ check(rval, avro_schema_to_json2(val.schema, out,
parent_namespace));
}
return avro_write_str(out, "]");
}
-static int write_link(avro_writer_t out, const struct avro_link_schema_t *link)
+static int write_link(avro_writer_t out, const struct avro_link_schema_t *link,
+ const char *parent_namespace)
{
int rval;
check(rval, avro_write_str(out, "\""));
+ if (is_avro_record(link->to)) {
+ const char *namespace = avro_schema_to_record(link->to)->space;
+ if (nullstrcmp(namespace, parent_namespace)) {
+ check(rval, avro_write_str(out, namespace));
+ check(rval, avro_write_str(out, "."));
+ }
+ }
check(rval, avro_write_str(out, avro_schema_name(link->to)));
return avro_write_str(out, "\"");
}
-int avro_schema_to_json(const avro_schema_t schema, avro_writer_t out)
+static int
+avro_schema_to_json2(const avro_schema_t schema, avro_writer_t out,
+ const char *parent_namespace)
{
check_param(EINVAL, is_avro_schema(schema), "schema");
check_param(EINVAL, out, "writer");
@@ -1625,19 +1674,19 @@ int avro_schema_to_json(const avro_schem
check(rval, avro_write_str(out, "null"));
break;
case AVRO_RECORD:
- return write_record(out, avro_schema_to_record(schema));
+ return write_record(out, avro_schema_to_record(schema),
parent_namespace);
case AVRO_ENUM:
return write_enum(out, avro_schema_to_enum(schema));
case AVRO_FIXED:
return write_fixed(out, avro_schema_to_fixed(schema));
case AVRO_MAP:
- return write_map(out, avro_schema_to_map(schema));
+ return write_map(out, avro_schema_to_map(schema),
parent_namespace);
case AVRO_ARRAY:
- return write_array(out, avro_schema_to_array(schema));
+ return write_array(out, avro_schema_to_array(schema),
parent_namespace);
case AVRO_UNION:
- return write_union(out, avro_schema_to_union(schema));
+ return write_union(out, avro_schema_to_union(schema),
parent_namespace);
case AVRO_LINK:
- return write_link(out, avro_schema_to_link(schema));
+ return write_link(out, avro_schema_to_link(schema),
parent_namespace);
}
if (is_avro_primitive(schema)) {
@@ -1646,3 +1695,8 @@ int avro_schema_to_json(const avro_schem
avro_set_error("Unknown schema type");
return EINVAL;
}
+
+int avro_schema_to_json(const avro_schema_t schema, avro_writer_t out)
+{
+ return avro_schema_to_json2(schema, out, NULL);
+}
Modified: avro/trunk/lang/c/src/schema_equal.c
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/src/schema_equal.c?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/src/schema_equal.c (original)
+++ avro/trunk/lang/c/src/schema_equal.c Fri May 24 13:49:20 2013
@@ -30,13 +30,7 @@ schema_record_equal(struct avro_record_s
*/
return 0;
}
- if (a->space && b->space) {
- /* They have different namespaces */
- if (strcmp(a->space, b->space)) {
- return 0;
- }
- } else if (a->space || b->space) {
- /* One has a namespace, one doesn't */
+ if (nullstrcmp(a->space, b->space)) {
return 0;
}
for (i = 0; i < a->fields->num_entries; i++) {
@@ -148,6 +142,15 @@ schema_link_equal(struct avro_link_schem
* recursive schemas so we just check the name of the schema pointed
* to instead of a deep check. Otherwise, we recurse forever...
*/
+ if (is_avro_record(a->to)) {
+ if (!is_avro_record(b->to)) {
+ return 0;
+ }
+ if (nullstrcmp(avro_schema_to_record(a->to)->space,
+ avro_schema_to_record(b->to)->space)) {
+ return 0;
+ }
+ }
return (strcmp(avro_schema_name(a->to), avro_schema_name(b->to)) == 0);
}
Added: avro/trunk/lang/c/tests/schema_tests/pass/namespace_recursive
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/schema_tests/pass/namespace_recursive?rev=1486053&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/schema_tests/pass/namespace_recursive (added)
+++ avro/trunk/lang/c/tests/schema_tests/pass/namespace_recursive Fri May 24
13:49:20 2013
@@ -0,0 +1,28 @@
+{ "type": "record",
+ "name": "Container",
+ "namespace": "namespace1",
+ "fields": [
+ { "name": "contained",
+ "type": { "type": "record",
+ "name": "MutuallyRecursive",
+ "fields": [
+ { "name": "label", "type": "string" },
+ { "name": "children",
+ "type": {"type": "array", "items":
+ {"type": "record",
+ "name": "MutuallyRecursive",
+ "namespace": "namespace2",
+ "fields": [
+ { "name": "value", "type": "int" },
+ { "name": "children", "type": {"type":
"array", "items": "namespace1.MutuallyRecursive" }},
+ { "name": "morechildren", "type": {"type":
"array", "items": "MutuallyRecursive" }}
+ ]
+ }
+ }
+ },
+ { "name": "anotherchild", "type":
"namespace2.MutuallyRecursive"}
+ ]
+ }
+ }
+ ]
+}
Added: avro/trunk/lang/c/tests/schema_tests/pass/namespace_simple
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/schema_tests/pass/namespace_simple?rev=1486053&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/schema_tests/pass/namespace_simple (added)
+++ avro/trunk/lang/c/tests/schema_tests/pass/namespace_simple Fri May 24
13:49:20 2013
@@ -0,0 +1,5 @@
+{"type": "record", "namespace": "x", "name": "Y", "fields": [
+ {"name": "e", "type": {"type": "record", "name": "Z", "fields": [
+ {"name": "f", "type": "x.Z"}
+ ]}}
+]}
Modified: avro/trunk/lang/c/tests/test_avro_schema.c
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/test_avro_schema.c?rev=1486053&r1=1486052&r2=1486053&view=diff
==============================================================================
--- avro/trunk/lang/c/tests/test_avro_schema.c (original)
+++ avro/trunk/lang/c/tests/test_avro_schema.c Fri May 24 13:49:20 2013
@@ -33,12 +33,14 @@ avro_writer_t avro_stderr;
static void run_tests(char *dirpath, int should_pass)
{
char jsontext[4096];
+ char jsontext2[4096];
size_t rval;
char filepath[1024];
DIR *dir;
struct dirent *dent;
FILE *fp;
avro_schema_t schema;
+ avro_writer_t jsontext2_writer;
dir = opendir(dirpath);
if (dir == NULL) {
@@ -82,6 +84,26 @@ static void run_tests(char *dirpath, int
"failed to
avro_schema_equal(schema,avro_schema_copy())\n");
exit(EXIT_FAILURE);
}
+ jsontext2_writer =
avro_writer_memory(jsontext2, sizeof(jsontext2));
+ if (avro_schema_to_json(schema,
jsontext2_writer)) {
+ fprintf(stderr, "failed to
write schema (%s)\n",
+ avro_strerror());
+ exit(EXIT_FAILURE);
+ }
+ avro_write(jsontext2_writer, (void
*)"", 1); /* zero terminate */
+ avro_writer_free(jsontext2_writer);
+ avro_schema_decref(schema);
+ if (avro_schema_from_json(jsontext2, 0,
&schema, NULL)) {
+ fprintf(stderr, "failed to
write then read schema (%s)\n",
+ avro_strerror());
+ exit(EXIT_FAILURE);
+ }
+ if (!avro_schema_equal
+ (schema, schema_copy)) {
+ fprintf(stderr, "failed
read-write-read cycle (%s)\n",
+ avro_strerror());
+ exit(EXIT_FAILURE);
+ }
avro_schema_decref(schema_copy);
avro_schema_decref(schema);
} else {
@@ -94,6 +116,7 @@ static void run_tests(char *dirpath, int
}
} else {
if (should_pass) {
+ fprintf(stderr, "%s\n",
avro_strerror());
fprintf(stderr,
"fail! (should have succeeded
but didn't)\n");
exit(EXIT_FAILURE);
