Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/16405#discussion_r94273331
--- Diff: dev/lint-python ---
@@ -19,10 +19,8 @@
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
-PATHS_TO_CHECK="./python/pyspark/ ./examples/src/main/python/
./dev/sparktestsupport"
-# TODO: fix pep8 errors with the rest of the Python scripts under dev
-PATHS_TO_CHECK="$PATHS_TO_CHECK ./dev/run-tests.py ./python/*.py
./dev/run-tests-jenkins.py"
-PATHS_TO_CHECK="$PATHS_TO_CHECK ./dev/pip-sanity-check.py"
+# Exclude auto-geneated configuration file.
+PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" -not -path
"*python/docs/conf.py" )"
--- End diff --
I tested this as below for sure,
```bash
./lint-python
./dev/lint-python
./spark/dev/lint-python
```
So, now it is relative paths which currently are up to 11K as below:
```
./dev/create-release/generate-contributors.py
./dev/create-release/releaseutils.py
./dev/create-release/translate-contributors.py ./dev/github_jira_sync.py
./dev/merge_spark_pr.py ./dev/pep8-1.7.0.py ./dev/pip-sanity-check.py
./dev/run-tests-jenkins.py ./dev/run-tests.py
./dev/sparktestsupport/__init__.py ./dev/sparktestsupport/modules.py
./dev/sparktestsupport/shellutils.py ./dev/sparktestsupport/toposort.py
./examples/src/main/python/als.py
./examples/src/main/python/avro_inputformat.py
./examples/src/main/python/kmeans.py
./examples/src/main/python/logistic_regression.py
./examples/src/main/python/ml/aft_survival_regression.py
./examples/src/main/python/ml/als_example.py
./examples/src/main/python/ml/binarizer_example.py
./examples/src/main/python/ml/bisecting_k_means_example.py
./examples/src/main/python/ml/bucketizer_example.py
./examples/src/main/python/ml/chisq_selector_example.py
./examples/src/main/python/ml/count_vectorizer_example.py
./examples/src/main/python/ml/cross
_validator.py ./examples/src/main/python/ml/dataframe_example.py
./examples/src/main/python/ml/dct_example.py
./examples/src/main/python/ml/decision_tree_classification_example.py
./examples/src/main/python/ml/decision_tree_regression_example.py
./examples/src/main/python/ml/elementwise_product_example.py
./examples/src/main/python/ml/estimator_transformer_param_example.py
./examples/src/main/python/ml/gaussian_mixture_example.py
./examples/src/main/python/ml/generalized_linear_regression_example.py
./examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
./examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
./examples/src/main/python/ml/index_to_string_example.py
./examples/src/main/python/ml/isotonic_regression_example.py
./examples/src/main/python/ml/kmeans_example.py
./examples/src/main/python/ml/lda_example.py
./examples/src/main/python/ml/linear_regression_with_elastic_net.py
./examples/src/main/python/ml/logistic_regression_summary_example.
py ./examples/src/main/python/ml/logistic_regression_with_elastic_net.py
./examples/src/main/python/ml/max_abs_scaler_example.py
./examples/src/main/python/ml/min_max_scaler_example.py
./examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
./examples/src/main/python/ml/multilayer_perceptron_classification.py
./examples/src/main/python/ml/n_gram_example.py
./examples/src/main/python/ml/naive_bayes_example.py
./examples/src/main/python/ml/normalizer_example.py
./examples/src/main/python/ml/one_vs_rest_example.py
./examples/src/main/python/ml/onehot_encoder_example.py
./examples/src/main/python/ml/pca_example.py
./examples/src/main/python/ml/pipeline_example.py
./examples/src/main/python/ml/polynomial_expansion_example.py
./examples/src/main/python/ml/quantile_discretizer_example.py
./examples/src/main/python/ml/random_forest_classifier_example.py
./examples/src/main/python/ml/random_forest_regressor_example.py
./examples/src/main/python/ml/rformula_example.py
./examples/src/main/python/ml/sql_transformer.py
./examples/src/main/python/ml/standard_scaler_example.py
./examples/src/main/python/ml/stopwords_remover_example.py
./examples/src/main/python/ml/string_indexer_example.py
./examples/src/main/python/ml/tf_idf_example.py
./examples/src/main/python/ml/tokenizer_example.py
./examples/src/main/python/ml/train_validation_split.py
./examples/src/main/python/ml/vector_assembler_example.py
./examples/src/main/python/ml/vector_indexer_example.py
./examples/src/main/python/ml/vector_slicer_example.py
./examples/src/main/python/ml/word2vec_example.py
./examples/src/main/python/mllib/binary_classification_metrics_example.py
./examples/src/main/python/mllib/bisecting_k_means_example.py
./examples/src/main/python/mllib/correlations.py
./examples/src/main/python/mllib/correlations_example.py
./examples/src/main/python/mllib/decision_tree_classification_example.py
./examples/src/main/python/mllib/decision_tree_regression_example.py
./examples/src/ma
in/python/mllib/elementwise_product_example.py
./examples/src/main/python/mllib/fpgrowth_example.py
./examples/src/main/python/mllib/gaussian_mixture_example.py
./examples/src/main/python/mllib/gaussian_mixture_model.py
./examples/src/main/python/mllib/gradient_boosting_classification_example.py
./examples/src/main/python/mllib/gradient_boosting_regression_example.py
./examples/src/main/python/mllib/hypothesis_testing_example.py
./examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
./examples/src/main/python/mllib/isotonic_regression_example.py
./examples/src/main/python/mllib/k_means_example.py
./examples/src/main/python/mllib/kernel_density_estimation_example.py
./examples/src/main/python/mllib/kmeans.py
./examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
./examples/src/main/python/mllib/linear_regression_with_sgd_example.py
./examples/src/main/python/mllib/logistic_regression.py
./examples/src/main/python/mllib/logistic_regress
ion_with_lbfgs_example.py
./examples/src/main/python/mllib/multi_class_metrics_example.py
./examples/src/main/python/mllib/multi_label_metrics_example.py
./examples/src/main/python/mllib/naive_bayes_example.py
./examples/src/main/python/mllib/normalizer_example.py
./examples/src/main/python/mllib/power_iteration_clustering_example.py
./examples/src/main/python/mllib/random_forest_classification_example.py
./examples/src/main/python/mllib/random_forest_regression_example.py
./examples/src/main/python/mllib/random_rdd_generation.py
./examples/src/main/python/mllib/ranking_metrics_example.py
./examples/src/main/python/mllib/recommendation_example.py
./examples/src/main/python/mllib/regression_metrics_example.py
./examples/src/main/python/mllib/sampled_rdds.py
./examples/src/main/python/mllib/standard_scaler_example.py
./examples/src/main/python/mllib/stratified_sampling_example.py
./examples/src/main/python/mllib/streaming_k_means_example.py
./examples/src/main/python/mllib/streaming_l
inear_regression_example.py
./examples/src/main/python/mllib/summary_statistics_example.py
./examples/src/main/python/mllib/svm_with_sgd_example.py
./examples/src/main/python/mllib/tf_idf_example.py
./examples/src/main/python/mllib/word2vec.py
./examples/src/main/python/mllib/word2vec_example.py
./examples/src/main/python/pagerank.py
./examples/src/main/python/parquet_inputformat.py
./examples/src/main/python/pi.py ./examples/src/main/python/sort.py
./examples/src/main/python/sql/basic.py
./examples/src/main/python/sql/datasource.py
./examples/src/main/python/sql/hive.py
./examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
./examples/src/main/python/sql/streaming/structured_network_wordcount.py
./examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
./examples/src/main/python/status_api_demo.py
./examples/src/main/python/streaming/direct_kafka_wordcount.py
./examples/src/main/python/streaming/flume_wordcount.py
./examples/src/main/python/s
treaming/hdfs_wordcount.py
./examples/src/main/python/streaming/kafka_wordcount.py
./examples/src/main/python/streaming/network_wordcount.py
./examples/src/main/python/streaming/network_wordjoinsentiments.py
./examples/src/main/python/streaming/queue_stream.py
./examples/src/main/python/streaming/recoverable_network_wordcount.py
./examples/src/main/python/streaming/sql_network_wordcount.py
./examples/src/main/python/streaming/stateful_network_wordcount.py
./examples/src/main/python/transitive_closure.py
./examples/src/main/python/wordcount.py
./external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
./python/docs/epytext.py ./python/pyspark/__init__.py
./python/pyspark/accumulators.py ./python/pyspark/broadcast.py
./python/pyspark/cloudpickle.py ./python/pyspark/conf.py
./python/pyspark/context.py ./python/pyspark/daemon.py
./python/pyspark/files.py ./python/pyspark/find_spark_home.py
./python/pyspark/heapq3.py ./python/pyspark/java_gateway.py ./python/pyspa
rk/join.py ./python/pyspark/ml/__init__.py ./python/pyspark/ml/base.py
./python/pyspark/ml/classification.py ./python/pyspark/ml/clustering.py
./python/pyspark/ml/common.py ./python/pyspark/ml/evaluation.py
./python/pyspark/ml/feature.py ./python/pyspark/ml/linalg/__init__.py
./python/pyspark/ml/param/__init__.py
./python/pyspark/ml/param/_shared_params_code_gen.py
./python/pyspark/ml/param/shared.py ./python/pyspark/ml/pipeline.py
./python/pyspark/ml/recommendation.py ./python/pyspark/ml/regression.py
./python/pyspark/ml/tests.py ./python/pyspark/ml/tuning.py
./python/pyspark/ml/util.py ./python/pyspark/ml/wrapper.py
./python/pyspark/mllib/__init__.py ./python/pyspark/mllib/classification.py
./python/pyspark/mllib/clustering.py ./python/pyspark/mllib/common.py
./python/pyspark/mllib/evaluation.py ./python/pyspark/mllib/feature.py
./python/pyspark/mllib/fpm.py ./python/pyspark/mllib/linalg/__init__.py
./python/pyspark/mllib/linalg/distributed.py ./python/pyspark/mllib/random.py
./py
thon/pyspark/mllib/recommendation.py ./python/pyspark/mllib/regression.py
./python/pyspark/mllib/stat/__init__.py
./python/pyspark/mllib/stat/_statistics.py
./python/pyspark/mllib/stat/distribution.py
./python/pyspark/mllib/stat/KernelDensity.py
./python/pyspark/mllib/stat/test.py ./python/pyspark/mllib/tests.py
./python/pyspark/mllib/tree.py ./python/pyspark/mllib/util.py
./python/pyspark/profiler.py ./python/pyspark/rdd.py
./python/pyspark/rddsampler.py ./python/pyspark/resultiterable.py
./python/pyspark/serializers.py ./python/pyspark/shell.py
./python/pyspark/shuffle.py ./python/pyspark/sql/__init__.py
./python/pyspark/sql/catalog.py ./python/pyspark/sql/column.py
./python/pyspark/sql/conf.py ./python/pyspark/sql/context.py
./python/pyspark/sql/dataframe.py ./python/pyspark/sql/functions.py
./python/pyspark/sql/group.py ./python/pyspark/sql/readwriter.py
./python/pyspark/sql/session.py ./python/pyspark/sql/streaming.py
./python/pyspark/sql/tests.py ./python/pyspark/sql/types.py
./python/pyspark/sql/utils.py ./python/pyspark/sql/window.py
./python/pyspark/statcounter.py ./python/pyspark/status.py
./python/pyspark/storagelevel.py ./python/pyspark/streaming/__init__.py
./python/pyspark/streaming/context.py ./python/pyspark/streaming/dstream.py
./python/pyspark/streaming/flume.py ./python/pyspark/streaming/kafka.py
./python/pyspark/streaming/kinesis.py ./python/pyspark/streaming/listener.py
./python/pyspark/streaming/tests.py ./python/pyspark/streaming/util.py
./python/pyspark/taskcontext.py ./python/pyspark/tests.py
./python/pyspark/traceback_utils.py ./python/pyspark/version.py
./python/pyspark/worker.py ./python/run-tests.py ./python/setup.py
./python/test_support/SimpleHTTPServer.py ./python/test_support/userlibrary.py
./sql/hive/src/test/resources/data/scripts/cat.py
./sql/hive/src/test/resources/data/scripts/cat_error.py
./sql/hive/src/test/resources/data/scripts/doubleescapedtab.py
./sql/hive/src/test/resources/data/scripts/dumpdata_script.py ./sql/hive
/src/test/resources/data/scripts/escapedcarriagereturn.py
./sql/hive/src/test/resources/data/scripts/escapednewline.py
./sql/hive/src/test/resources/data/scripts/escapedtab.py
./sql/hive/src/test/resources/data/scripts/input20_script.py
./sql/hive/src/test/resources/data/scripts/newline.py
./sql/hive/src/test/resources/data/scripts/test_transform.py
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
