??????source
my_env/bin/activate??????????????PYFLINK_CLIENT_EXECUTABLE??????????????????????????
????????????jobmanager????????????No module named
pyflink????jobmanager????????yarn????????
??????????????????????????
> LogType:jobmanager.out
> Log Upload Time:?????? ?????? 18 20:48:45 +0800 2021
> LogLength:37
> Log Contents:
> /bin/python: No module named pyflink
------------------ ???????? ------------------
??????:
"user-zh"
<[email protected]>;
????????: 2021??11??19??(??????) ????9:38
??????: "user-zh"<[email protected]>;
????: Re: ??????????????downloads/setup-pyflink-virtual-env.sh????
-pyexec
????????????????????Python????????????????????Flink??????????????Python????????????????????????:
https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/dev/python/dependency_management/#python-interpreter-of-client
On Thu, Nov 18, 2021 at 9:00 PM Asahi Lee <[email protected]> wrote:
> Hi !
> &nbsp; &nbsp;????java Table api??????python udf
>
????????????????????????????????????????python????????????????????????????????jm??????/bin/python:
No module named
> pyflink??
>
>
> ./flink-1.13.2/bin/flink&nbsp;
> run-application -t yarn-application&nbsp;
>
-Dyarn.provided.lib.dirs="hdfs://nameservice1/user/flink/flinklib"&nbsp;
> -Dyarn.application.queue=d
> -p 1&nbsp;
> -pyarch /opt/venv.zip
> -pyexec venv.zip/venv/bin/python&nbsp;
> -pyfs /opt/test.py&nbsp;
> -c test.PyUDFTest&nbsp;
> /opt/flink-python-test-1.0-SNAPSHOT.jar
>
>
>
> ??????
> Caused by: java.lang.RuntimeException: Python callback server start failed!
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.python.PythonFunctionFactory.createPythonFunctionFactory(PythonFunctionFactory.java:167)
> ~[flink-python_2.11-1.13.2.jar:1.13.2]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.python.PythonFunctionFactory$1.load(PythonFunctionFactory.java:88)
> ~[flink-python_2.11-1.13.2.jar:1.13.2]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.python.PythonFunctionFactory$1.load(PythonFunctionFactory.java:84)
> ~[flink-python_2.11-1.13.2.jar:1.13.2]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2319)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2282)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2197)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache.get(LocalCache.java:3937)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3941)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.shaded.guava18.com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4824)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.python.PythonFunctionFactory.getPythonFunction(PythonFunctionFactory.java:129)
> ~[flink-python_2.11-1.13.2.jar:1.13.2]
> &nbsp; &nbsp; &nbsp; &nbsp; at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
> java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.functions.python.utils.PythonFunctionUtils.getPythonFunction(PythonFunctionUtils.java:45)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.functions.UserDefinedFunctionHelper.instantiateFunction(UserDefinedFunctionHelper.java:206)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.catalog.FunctionCatalog.getFunctionDefinition(FunctionCatalog.java:659)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.catalog.FunctionCatalog.resolveAmbiguousFunctionReference(FunctionCatalog.java:606)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.catalog.FunctionCatalog.lookupFunction(FunctionCatalog.java:362)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.planner.catalog.FunctionCatalogOperatorTable.lookupOperatorOverloads(FunctionCatalogOperatorTable.java:97)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.util.ChainedSqlOperatorTable.lookupOperatorOverloads(ChainedSqlOperatorTable.java:67)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1183)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1169)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1200)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1169)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:945)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:704)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.planner.calcite.FlinkPlannerImpl.org$apache$flink$table$planner$calcite$FlinkPlannerImpl$$validate(FlinkPlannerImpl.scala:152)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.planner.calcite.FlinkPlannerImpl.validate(FlinkPlannerImpl.scala:110)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.planner.operations.SqlToOperationConverter.convert(SqlToOperationConverter.java:201)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:101)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.table.api.internal.TableEnvironmentImpl.sqlQuery(TableEnvironmentImpl.java:704)
> ~[flink-table-blink_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
test.PyUDFTest.main(PyUDFTest.java:22)
> ~[flink-python-test-1.0-SNAPSHOT.jar:?]
> &nbsp; &nbsp; &nbsp; &nbsp; at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
> java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_111]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
> org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; at
>
org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:242)
> ~[flink-dist_2.11-1.13.1.jar:1.13.1]
> &nbsp; &nbsp; &nbsp; &nbsp; ... 10 more
> 2021-11-18 20:48:44,458 INFO&nbsp;
> org.apache.flink.runtime.blob.BlobServer&nbsp; &nbsp; &nbsp;
&nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;[] -
Stopped BLOB server at
> 0.0.0.0:45070
> 2021-11-18 20:48:44,458 INFO&nbsp;
> org.apache.flink.runtime.entrypoint.ClusterEntrypoint&nbsp; &nbsp;
&nbsp;
> &nbsp; [] - Shutting YarnApplicationClusterEntryPoint down with
application
> status UNKNOWN. Diagnostics Cluster entrypoint has been closed externally..
> 2021-11-18 20:48:44,458 INFO&nbsp;
> org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint [] - Shutting
> down rest endpoint.
> 2021-11-18 20:48:44,474 INFO&nbsp;
> org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint [] - Removing
> cache directory
> /tmp/flink-web-dd82c3c0-f457-492d-8e64-5ae74fe9abbd/flink-web-ui
> 2021-11-18 20:48:44,474 INFO&nbsp;
> org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint [] -
> http://cdh5node3:40216 lost leadership
> 2021-11-18 20:48:44,474 INFO&nbsp;
> org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint [] - Shut
> down complete.
> 2021-11-18 20:48:44,474 INFO&nbsp;
>
org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent
> [] - Closing components.
> 2021-11-18 20:48:44,475 INFO&nbsp;
> org.apache.flink.runtime.dispatcher.runner.SessionDispatcherLeaderProcess
> [] - Stopping SessionDispatcherLeaderProcess.
> 2021-11-18 20:48:44,475 INFO&nbsp;
> org.apache.flink.runtime.dispatcher.StandaloneDispatcher&nbsp;
&nbsp;
> &nbsp;[] - Stopping dispatcher akka.tcp://flink@cdh5node3
> :34697/user/rpc/dispatcher_1.
> 2021-11-18 20:48:44,476 INFO&nbsp;
> org.apache.flink.runtime.dispatcher.StandaloneDispatcher&nbsp;
&nbsp;
> &nbsp;[] - Stopping all currently running jobs of dispatcher
> akka.tcp://flink@cdh5node3:34697/user/rpc/dispatcher_1.
>
>
> LogType:jobmanager.out
> Log Upload Time:?????? ?????? 18 20:48:45 +0800 2021
> LogLength:37
> Log Contents:
> /bin/python: No module named pyflink
>
>
>
>
>
>
>
>
>
>
>
>
>
> ------------------&nbsp;????????&nbsp;------------------
> ??????:
>
"user-zh"
>
<
> [email protected]&gt;;
> ????????:&nbsp;2021??11??18??(??????) ????3:34
> ??????:&nbsp;"user-zh"<[email protected]&gt;;
>
> ????:&nbsp;Re:??????????????downloads/setup-pyflink-virtual-env.sh????
>
>
>
> Hi!<br/&gt;??????????????????????????????????????????<br/&gt;
>
https://nightlies.apache.org/flink/flink-docs-release-1.12/downloads/setup-pyflink-virtual-env.sh
> ?? 2021-11-18 15:05:03??"Asahi Lee" <[email protected]&gt;
??????
> &gt;Hi!
> &gt;&amp;nbsp; &amp;nbsp;
>
????flink??????????????????&amp;nbsp;&amp;nbsp;setup-pyflink-virtual-env.sh
> ????????python??????????????????????????????????????????????
> &gt;
>
https://nightlies.apache.org/flink/flink-docs-release-1.14/zh/docs/dev/python/faq/
