ibzib commented on a change in pull request #15712: URL: https://github.com/apache/beam/pull/15712#discussion_r731398099
########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionConsumer.java ########## @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A {@link ProjectionConsumer} is a {@link Schema}-aware operation (such as a {@link + * org.apache.beam.sdk.transforms.DoFn} or {@link org.apache.beam.sdk.transforms.PTransform}) that + * has a {@link org.apache.beam.sdk.schemas.FieldAccessDescriptor} describing which fields the + * operation accesses. + */ +@Experimental +public interface ProjectionConsumer { + /** + * Returns a {@link org.apache.beam.sdk.schemas.FieldAccessDescriptor} describing which Schema + * fields this operation must access to complete successfully. Fields not listed in the {@link + * org.apache.beam.sdk.schemas.FieldAccessDescriptor} are assumed to be safe to drop from the + * input. By default, the operation is assumed to access all fields. + */ + default FieldAccessDescriptor consumesProjection() { Review comment: I removed the default method. ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { + /** What kinds of projection support (if any) an operation offers. */ + enum ProjectSupport { + /** + * No projections are supported. In other words, the operation can only return a fixed set of + * transforms from its input. + */ + NONE, + /** + * Projections are supported as long as fields are projected in the same order as the data + * source. + */ + WITHOUT_FIELD_REORDERING, + /** All projections are supported. */ + WITH_FIELD_REORDERING + } + + /** What kinds of projection support (if any) this operation offers. Default: NONE */ + default ProjectSupport supportsProjectionPushdown() { + return ProjectSupport.NONE; + } + + /** Returns an operation that actuates the projection described by {@code fields}. */ + default T actuateProjectionPushdown(FieldAccessDescriptor fields) { + throw new UnsupportedOperationException(); Review comment: I removed the default methods. ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { + /** What kinds of projection support (if any) an operation offers. */ + enum ProjectSupport { + /** + * No projections are supported. In other words, the operation can only return a fixed set of + * transforms from its input. + */ + NONE, + /** + * Projections are supported as long as fields are projected in the same order as the data + * source. + */ + WITHOUT_FIELD_REORDERING, + /** All projections are supported. */ + WITH_FIELD_REORDERING + } + + /** What kinds of projection support (if any) this operation offers. Default: NONE */ + default ProjectSupport supportsProjectionPushdown() { Review comment: Done. ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { + /** What kinds of projection support (if any) an operation offers. */ + enum ProjectSupport { + /** + * No projections are supported. In other words, the operation can only return a fixed set of + * transforms from its input. + */ + NONE, Review comment: Done. ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { Review comment: I wrote this assumption into the javadoc. ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { + /** What kinds of projection support (if any) an operation offers. */ + enum ProjectSupport { + /** + * No projections are supported. In other words, the operation can only return a fixed set of + * transforms from its input. + */ + NONE, + /** + * Projections are supported as long as fields are projected in the same order as the data + * source. + */ + WITHOUT_FIELD_REORDERING, Review comment: Yes, specifically BigQuery's "direct read" path: https://github.com/apache/beam/blob/70ac565ea71f247fbe545489a5916ed058f4ddac/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryTable.java#L201 ########## File path: sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/ProjectionProducer.java ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.schemas; + +import org.apache.beam.sdk.annotations.Experimental; + +/** + * A factory for operations that execute a projection on a {@link Schema}-aware {@link + * org.apache.beam.sdk.values.PCollection}. + * + * <p>Typically this interface will be implemented by a reader {@link + * org.apache.beam.sdk.transforms.PTransform} that is capable of pushing down a projection to an + * external source. + */ +@Experimental +public interface ProjectionProducer<T> { + /** What kinds of projection support (if any) an operation offers. */ + enum ProjectSupport { + /** + * No projections are supported. In other words, the operation can only return a fixed set of + * transforms from its input. Review comment: I meant "fields." I corrected it. ########## File path: sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/SchemaIOTableProviderWrapper.java ########## @@ -153,10 +154,16 @@ public Schema getSchema() { @Override public ProjectSupport supportsProjects() { PTransform<PBegin, PCollection<Row>> readerTransform = schemaIO.buildReader(); - if (readerTransform instanceof PushdownProjector) { - return ((PushdownProjector) readerTransform).supportsFieldReordering() - ? ProjectSupport.WITH_FIELD_REORDERING - : ProjectSupport.WITHOUT_FIELD_REORDERING; + if (readerTransform instanceof ProjectionProducer) { + switch (((ProjectionProducer<?>) readerTransform).supportsProjectionPushdown()) { Review comment: SQL has a corresponding but separate `ProjectSupport` enum. https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/ProjectSupport.java -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
