EBernhardson has uploaded a new change for review. https://gerrit.wikimedia.org/r/260030
Change subject: Update avro schemas to use event-schema repo as submodule ...................................................................... Update avro schemas to use event-schema repo as submodule To keep a single source of truth about what schemas exist, the event-schema repo was created. This patch adjusts camus to utilize that repo rather than it's own copy of the schemas. Change-Id: I93c85205bab99ead70529fa80da3503a65d08dff --- A .gitmodules M refinery-camus/pom.xml M refinery-camus/src/main/java/org/wikimedia/analytics/refinery/camus/schemaregistry/LocalRepoSchemaRegistry.java D refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/101446746400.avsc D refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/111448028943.avsc A refinery-camus/src/main/resources/schema_repo M refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageDecoder.java M refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageTimestamps.java M refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroSchemaEvolution.java R refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/0.avsc R refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/1.avsc R refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/2.avsc R refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestTimestampSchema/0.avsc 13 files changed, 11 insertions(+), 314 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery/source refs/changes/30/260030/1 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1ed1f66 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "refinery-camus/src/main/resources/schema_repo"] + path = refinery-camus/src/main/resources/schema_repo + url = https://gerrit.wikimedia.org/r/mediawiki/event-schemas diff --git a/refinery-camus/pom.xml b/refinery-camus/pom.xml index b37f071..9289cfd 100644 --- a/refinery-camus/pom.xml +++ b/refinery-camus/pom.xml @@ -84,7 +84,7 @@ <goal>schema</goal> </goals> <configuration> - <sourceDirectory>${project.basedir}/src/test/resources/avro_schema_repo</sourceDirectory> + <sourceDirectory>${project.basedir}/src/test/resources/schema_repo/avro</sourceDirectory> <outputDirectory>${project.basedir}/src/generated/java/</outputDirectory> </configuration> </execution> diff --git a/refinery-camus/src/main/java/org/wikimedia/analytics/refinery/camus/schemaregistry/LocalRepoSchemaRegistry.java b/refinery-camus/src/main/java/org/wikimedia/analytics/refinery/camus/schemaregistry/LocalRepoSchemaRegistry.java index 1ea0bc4..e3c983f 100644 --- a/refinery-camus/src/main/java/org/wikimedia/analytics/refinery/camus/schemaregistry/LocalRepoSchemaRegistry.java +++ b/refinery-camus/src/main/java/org/wikimedia/analytics/refinery/camus/schemaregistry/LocalRepoSchemaRegistry.java @@ -28,9 +28,9 @@ /** * Default path in classpath to lookup schemas, e.g.: - * /avro_schema_repo/${SchemaName}/${rev}.avsc + * /schema_repo/avro/mediawiki/${SchemaName}/${rev}.avsc */ - private static final String AVRO_SCHEMA_REPO_DEFAULT_PATH = "/avro_schema_repo"; + private static final String AVRO_SCHEMA_REPO_DEFAULT_PATH = "/schema_repo/avro/mediawiki"; public static final String SCHEMA_NAMESPACE = "org.wikimedia.analytics.schemas"; public static final String SCHEMA_REGISTRY_CLASS = "kafka.message.coder.schema.registry.class"; diff --git a/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/101446746400.avsc b/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/101446746400.avsc deleted file mode 100644 index b81c597..0000000 --- a/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/101446746400.avsc +++ /dev/null @@ -1,147 +0,0 @@ -{ - "type": "record", - "name": "CirrusSearchRequestSet", - "namespace": "org.wikimedia.analytics.schemas", - "doc": "A set of requests made by CirrusSearch to the elasticsearch user for a single php execution context", - "fields": [ - { - "name": "ts", - "doc": "The timestamp, in unix time, that the request was made", - "type": "int", - "default": 0 - }, - { - "name": "wikiId", - "doc": "The wiki making this request, such as dewiki or enwiktionary", - "type": "string", - "default": "" - }, - { - "name": "source", - "doc": "Where the request is coming from. Typically: web, api or cli", - "type": "string", - "default": "" - }, - { - "name": "identity", - "doc": "A hash identifying the requestor. Includes the IP address and User Agent when available.", - "type": "string", - "default": "" - }, - { - "name": "ip", - "doc": "The IP address (either ipv4 or ipv6) in string notation", - "type": "string", - "default": "" - }, - { - "name": "userAgent", - "doc": "The HTTP User-Agent header, or null if not-applicable", - "type": "string", - "default": "" - }, - { - "name": "backendUserTests", - "doc": "List of backend tests the requests are participating in", - "type": { - "type": "array", - "items": "string" - }, - "default": [] - }, - { - "name": "requests", - "doc": "A list of requests made between mediawiki and elasticsearch in a single execution context", - "default": [], - "type": { - "type": "array", - "items": { - "name": "CirrusSearchRequest", - "namespace": "org.wikimedia.analytics.schemas", - "doc": "An individual request made between mediawiki and elasticsearch", - "type": "record", - "fields": [ - { - "name": "query", - "doc": "The actual search request", - "type": "string", - "default": "" - }, - { - "name": "queryType", - "doc": "The general type of query performed, such as full_text, prefix, etc.", - "type": "string", - "default": "" - }, - { - "name": "indices", - "doc": "The list of indices the request was performed against", - "type": { - "type": "array", - "items": "string" - }, - "default": [] - }, - { - "name": "tookMs", - "doc": "The number of milliseconds between passing the query to the client library and getting the response back in the application", - "type": "int", - "default": -1 - }, - { - "name": "elasticTookMs", - "doc": "The number of milliseconds the query took, according to the elasticsearch response", - "type": "int", - "default": -1 - }, - { - "name": "limit", - "doc": "The maximum number of results requested by the application", - "type": "int", - "default": -1 - }, - { - "name": "hitsTotal", - "doc": "The approximate total number of documents matching the query", - "type": "int", - "default": -1 - }, - { - "name": "hitsReturned", - "doc": "The number of results returned to the application", - "type": "int", - "default": -1 - }, - { - "name": "hitsOffset", - "doc": "The offset of the query", - "type": "int", - "default": -1 - }, - { - "name": "namespaces", - "doc": "Each element is a mediawiki namespace id that was searched.", - "type": { - "type": "array", - "items": "int" - }, - "default": [] - }, - { - "name": "suggestion", - "doc": "The suggestion generated by elasticsearch", - "type": "string", - "default": "" - }, - { - "name": "suggestionRequested", - "doc": "If a suggestion was requested from elasticsearch", - "type": "boolean", - "default": false - } - ] - } - } - } - ] -} diff --git a/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/111448028943.avsc b/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/111448028943.avsc deleted file mode 100644 index 16c4525..0000000 --- a/refinery-camus/src/main/resources/avro_schema_repo/CirrusSearchRequestSet/111448028943.avsc +++ /dev/null @@ -1,159 +0,0 @@ -{ - "type": "record", - "name": "CirrusSearchRequestSet", - "namespace": "org.wikimedia.analytics.schemas", - "doc": "A set of requests made by CirrusSearch to the elasticsearch user for a single php execution context", - "fields": [ - { - "name": "ts", - "doc": "The timestamp, in unix time, that the request was made", - "type": "int", - "default": 0 - }, - { - "name": "wikiId", - "doc": "The wiki making this request, such as dewiki or enwiktionary", - "type": "string", - "default": "" - }, - { - "name": "source", - "doc": "Where the request is coming from. Typically: web, api or cli", - "type": "string", - "default": "" - }, - { - "name": "identity", - "doc": "A hash identifying the requestor. Includes the IP address and User Agent when available.", - "type": "string", - "default": "" - }, - { - "name": "ip", - "doc": "The IP address (either ipv4 or ipv6) in string notation", - "type": "string", - "default": "" - }, - { - "name": "userAgent", - "doc": "The HTTP User-Agent header, or null if not-applicable", - "type": "string", - "default": "" - }, - { - "name": "backendUserTests", - "doc": "List of backend tests the requests are participating in", - "type": { - "type": "array", - "items": "string" - }, - "default": [] - }, - { - "name": "payload", - "type": {"type": "map", "values": "string"}, - "doc": "General purpose data for this request set", - "default": {} - }, - { - "name": "requests", - "doc": "A list of requests made between mediawiki and elasticsearch in a single execution context", - "default": [], - "type": { - "type": "array", - "items": { - "name": "CirrusSearchRequest", - "namespace": "org.wikimedia.analytics.schemas", - "doc": "An individual request made between mediawiki and elasticsearch", - "type": "record", - "fields": [ - { - "name": "query", - "doc": "The actual search request", - "type": "string", - "default": "" - }, - { - "name": "queryType", - "doc": "The general type of query performed, such as full_text, prefix, etc.", - "type": "string", - "default": "" - }, - { - "name": "indices", - "doc": "The list of indices the request was performed against", - "type": { - "type": "array", - "items": "string" - }, - "default": [] - }, - { - "name": "tookMs", - "doc": "The number of milliseconds between passing the query to the client library and getting the response back in the application", - "type": "int", - "default": -1 - }, - { - "name": "elasticTookMs", - "doc": "The number of milliseconds the query took, according to the elasticsearch response", - "type": "int", - "default": -1 - }, - { - "name": "limit", - "doc": "The maximum number of results requested by the application", - "type": "int", - "default": -1 - }, - { - "name": "hitsTotal", - "doc": "The approximate total number of documents matching the query", - "type": "int", - "default": -1 - }, - { - "name": "hitsReturned", - "doc": "The number of results returned to the application", - "type": "int", - "default": -1 - }, - { - "name": "hitsOffset", - "doc": "The offset of the query", - "type": "int", - "default": -1 - }, - { - "name": "namespaces", - "doc": "Each element is a mediawiki namespace id that was searched.", - "type": { - "type": "array", - "items": "int" - }, - "default": [] - }, - { - "name": "suggestion", - "doc": "The suggestion generated by elasticsearch", - "type": "string", - "default": "" - }, - { - "name": "suggestionRequested", - "doc": "If a suggestion was requested from elasticsearch", - "type": "boolean", - "default": false - }, - { - "name": "payload", - "doc": "General purpose data for this request", - "type": {"type": "map", "values": "string"}, - "default": {} - } - ] - } - } - } - ] -} diff --git a/refinery-camus/src/main/resources/schema_repo b/refinery-camus/src/main/resources/schema_repo new file mode 160000 index 0000000..3ae2974 --- /dev/null +++ b/refinery-camus/src/main/resources/schema_repo +Subproject commit 3ae2974957d160ed11173252c4ddc4c4a26d5255 diff --git a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageDecoder.java b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageDecoder.java index 7aff60f..a876e12 100644 --- a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageDecoder.java +++ b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageDecoder.java @@ -49,7 +49,7 @@ } private static byte[] getAvroBinaryPayload() throws IOException { - Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/avro_schema_repo/TestSchema/0.avsc")); + Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/schema_repo/avro/mediawiki/TestSchema/0.avsc")); GenericRecordBuilder builder = new GenericRecordBuilder(s); builder.set("id", 1l); builder.set("name", "test name"); diff --git a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageTimestamps.java b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageTimestamps.java index cb9b029..ce5ef6f 100644 --- a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageTimestamps.java +++ b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroMessageTimestamps.java @@ -227,4 +227,4 @@ } } } -} \ No newline at end of file +} diff --git a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroSchemaEvolution.java b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroSchemaEvolution.java index c40ecdc..76a9701 100644 --- a/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroSchemaEvolution.java +++ b/refinery-camus/src/test/java/org/wikimedia/analytics/refinery/camus/coders/TestAvroSchemaEvolution.java @@ -237,7 +237,7 @@ } } private TestMessage getAvroBinaryPayloadV0() throws IOException { - Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/avro_schema_repo/TestSchema/0.avsc")); + Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/schema_repo/avro/mediawiki/TestSchema/0.avsc")); GenericRecordBuilder builder = new GenericRecordBuilder(s); builder.set("id", 1l); builder.set("name", "test name"); @@ -257,7 +257,7 @@ } private TestMessage getAvroBinaryPayloadV1() throws IOException { - Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/avro_schema_repo/TestSchema/1.avsc")); + Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/schema_repo/avro/mediawiki/TestSchema/1.avsc")); GenericRecordBuilder builder = new GenericRecordBuilder(s); builder.set("id", 1l); builder.set("name", "test name"); @@ -279,7 +279,7 @@ } private TestMessage getAvroBinaryPayloadV2() throws IOException { - Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/avro_schema_repo/TestSchema/2.avsc")); + Schema s = new Schema.Parser().parse(TestAvroSchemaEvolution.class.getResourceAsStream("/schema_repo/avro/mediawiki/TestSchema/2.avsc")); GenericRecordBuilder builder = new GenericRecordBuilder(s); builder.set("id", 1l); builder.set("name", "test name"); diff --git a/refinery-camus/src/test/resources/avro_schema_repo/TestSchema/0.avsc b/refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/0.avsc similarity index 100% rename from refinery-camus/src/test/resources/avro_schema_repo/TestSchema/0.avsc rename to refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/0.avsc diff --git a/refinery-camus/src/test/resources/avro_schema_repo/TestSchema/1.avsc b/refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/1.avsc similarity index 100% rename from refinery-camus/src/test/resources/avro_schema_repo/TestSchema/1.avsc rename to refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/1.avsc diff --git a/refinery-camus/src/test/resources/avro_schema_repo/TestSchema/2.avsc b/refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/2.avsc similarity index 100% rename from refinery-camus/src/test/resources/avro_schema_repo/TestSchema/2.avsc rename to refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestSchema/2.avsc diff --git a/refinery-camus/src/test/resources/avro_schema_repo/TestTimestampSchema/0.avsc b/refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestTimestampSchema/0.avsc similarity index 100% rename from refinery-camus/src/test/resources/avro_schema_repo/TestTimestampSchema/0.avsc rename to refinery-camus/src/test/resources/schema_repo/avro/mediawiki/TestTimestampSchema/0.avsc -- To view, visit https://gerrit.wikimedia.org/r/260030 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I93c85205bab99ead70529fa80da3503a65d08dff Gerrit-PatchSet: 1 Gerrit-Project: analytics/refinery/source Gerrit-Branch: master Gerrit-Owner: EBernhardson <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
