jihoonson commented on a change in pull request #7998: Binary license
management system
URL: https://github.com/apache/incubator-druid/pull/7998#discussion_r299744001
##########
File path: docs/_bin/generate-license-dependency-reports.py
##########
@@ -16,62 +16,85 @@
# limitations under the License.
import os
+import shutil
import subprocess
import sys
import argparse
+import concurrent.futures
+import time
+import threading
-existing_jar_dict_notice = {}
-def generate_reports(druid_path, tmp_path, exclude_ext):
- license_main_path = tmp_path + "/license-reports"
- license_ext_path = tmp_path + "/license-reports/ext"
- os.mkdir(license_main_path)
- os.mkdir(license_ext_path)
- print("********** Generating main LICENSE report.... **********")
- os.chdir(druid_path)
- command = "mvn -Pdist -Ddependency.locations.enabled=false
project-info-reports:dependencies"
- outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
- command = "cp -r distribution/target/site
{}/site".format(license_main_path)
- outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
- if exclude_ext:
- sys.exit()
+def generate_report(module_path, report_orig_path, report_out_path):
+ if not os.path.isdir(module_path):
+ print("{} is not a directory".format(module_path))
+ return
- print("********** Generating extension LICENSE reports.... **********")
- extension_dirs = os.listdir("extensions-core")
- print("Found {}".format(extension_dirs))
- for extension_dir in extension_dirs:
- full_extension_dir = druid_path + "/extensions-core/" + extension_dir
- if not os.path.isdir(full_extension_dir):
- print("{} is not a directory".format(full_extension_dir))
- continue
+ os.makedirs(report_out_path, exist_ok=True)
- print("--- Generating report for {}... ---".format(extension_dir))
+ try:
+ # This command prints lots of false errors. Here, we redirect stdout
and stderr to avoid them.
+ command = "mvn -Ddependency.locations.enabled=false
project-info-reports:dependencies"
+ subprocess.check_output(command, cwd=module_path, shell=True)
+ command = "cp -r {} {}".format(report_orig_path, report_out_path)
+ subprocess.check_output(command, cwd=module_path, shell=True)
+ except Exception as e:
+ print("Encountered error [{}] when generating report for {}".format(e,
module_path))
+
+
+def generate_reports(druid_path, tmp_path, exclude_ext, num_threads):
+ tmp_path = os.path.abspath(tmp_path)
+ license_report_root = os.path.join(tmp_path, "license-reports")
+ license_core_path = os.path.join(license_report_root, "core")
+ license_ext_path = os.path.join(license_report_root, "ext")
+ shutil.rmtree(license_report_root, ignore_errors=True)
+ os.makedirs(license_core_path)
+ os.makedirs(license_ext_path)
+ druid_path = os.path.abspath(druid_path)
+
+ script_args = [(druid_path, os.path.join(druid_path, "distribution",
"target", "site"), license_core_path)]
+
+ if not exclude_ext:
+ extensions_core_path = os.path.join(druid_path, "extensions-core")
+ extension_dirs = os.listdir(extensions_core_path)
+ print("Found {} extensions".format(len(extension_dirs)))
+ for extension_dir in extension_dirs:
+ extension_path = os.path.join(extensions_core_path, extension_dir)
+ if not os.path.isdir(extension_path):
+ print("{} is not a directory".format(extension_path))
+ continue
+
+ extension_report_dir = "{}/{}".format(license_ext_path,
extension_dir)
+ script_args.append((extension_path, os.path.join(extension_path,
"target", "site"), extension_report_dir))
+
+ print("Generating dependency reports", end="")
+ running = True
+ def print_dots():
+ while running:
+ print(".", end="", flush=True)
+ time.sleep(10)
+ dot_thread = threading.Thread(target=print_dots)
+ dot_thread.start()
- extension_report_dir = "{}/{}".format(license_ext_path, extension_dir)
- os.mkdir(extension_report_dir)
- prev_work_dir = os.getcwd()
- os.chdir(full_extension_dir)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as
executor:
+ for module_path, report_orig_path, report_out_path in script_args:
+ executor.submit(generate_report, module_path, report_orig_path,
report_out_path)
- try:
- command = "mvn -Ddependency.locations.enabled=false
project-info-reports:dependencies"
- outstr = subprocess.check_output(command,
shell=True).decode('UTF-8')
- command = "cp -r target/site {}/site".format(extension_report_dir)
- outstr = subprocess.check_output(command,
shell=True).decode('UTF-8')
- except:
- print("Encountered error when generating report for: " +
extension_dir)
+ running = False
+ dot_thread.join()
- os.chdir(prev_work_dir)
if __name__ == "__main__":
try:
parser = argparse.ArgumentParser(description='Generating dependency
reports.')
parser.add_argument('druid_path', metavar='<Druid source path>',
type=str)
parser.add_argument('tmp_path', metavar='<Full tmp path>', type=str)
Review comment:
This is not actually a tmp path, but the directory path where the dependency
reports are stored. I just kept its name as it was before.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
