[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Zequan Wu via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rGb380699416d8: [Utils] Add an option to specify number of 
cores to use in creduce-clang-crash. (authored by zequanwu).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

Files:
  clang/utils/creduce-clang-crash.py


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -19,6 +19,7 @@
 import tempfile
 import shutil
 from distutils.spawn import find_executable
+import multiprocessing
 
 verbose = False
 creduce_cmd = None
@@ -64,7 +65,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +79,7 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,9 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, 
+  default=max(4, multiprocessing.cpu_count() / 2),
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +406,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -19,6 +19,7 @@
 import tempfile
 import shutil
 from distutils.spawn import find_executable
+import multiprocessing
 
 verbose = False
 creduce_cmd = None
@@ -64,7 +65,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +79,7 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,9 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, 
+  default=max(4, multiprocessing.cpu_count() / 2),
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +406,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Reid Kleckner via Phabricator via cfe-commits]

rnk accepted this revision.
rnk added a comment.

lgtm


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Nico Weber via Phabricator via cfe-commits]

thakis accepted this revision.
thakis added a comment.
This revision is now accepted and ready to land.

But lgtm either way.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Nico Weber via Phabricator via cfe-commits]

thakis added a comment.

In D97098#2579620 , @zequanwu wrote:

> Address comment, using `max(4, multiprocessing.cpu_count() / 2)` because 
> `os.cpu_count` is not available in python2.

We require py3 these days, so it's ok to assume it.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Zequan Wu via Phabricator via cfe-commits]

zequanwu updated this revision to Diff 325520.
zequanwu added a comment.

Address comment, using `max(4, multiprocessing.cpu_count() / 2)` because 
`os.cpu_count` is not available in python2.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

Files:
  clang/utils/creduce-clang-crash.py


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -19,6 +19,7 @@
 import tempfile
 import shutil
 from distutils.spawn import find_executable
+import multiprocessing
 
 verbose = False
 creduce_cmd = None
@@ -64,7 +65,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +79,7 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,9 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, 
+  default=max(4, multiprocessing.cpu_count() / 2),
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +406,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -19,6 +19,7 @@
 import tempfile
 import shutil
 from distutils.spawn import find_executable
+import multiprocessing
 
 verbose = False
 creduce_cmd = None
@@ -64,7 +65,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +79,7 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,9 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, 
+  default=max(4, multiprocessing.cpu_count() / 2),
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +406,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Reid Kleckner via Phabricator via cfe-commits]

rnk added inline comments.



Comment at: clang/utils/creduce-clang-crash.py:81
 self.creduce_flags = ["--tidy"]
+if core_number > 0:
+  self.creduce_flags = ["--n", str(core_number)]

I think we should try to pick a good default here. I think we can do better 
than creduce's default of 4 or whatever it is. I think `max(4, os.cpu_count() / 
2)` might be a good guess.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D97098/new/

https://reviews.llvm.org/D97098

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D97098: [Utils] Add an option to specify number of cores to use in creduce-clang-crash.py

[Zequan Wu via Phabricator via cfe-commits]

zequanwu created this revision.
zequanwu added reviewers: akhuang, thakis, rnk.
zequanwu requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.

Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D97098

Files:
  clang/utils/creduce-clang-crash.py


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -64,7 +64,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +78,8 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+if core_number > 0:
+  self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,8 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, default=0,
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +405,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()


Index: clang/utils/creduce-clang-crash.py
===
--- clang/utils/creduce-clang-crash.py
+++ clang/utils/creduce-clang-crash.py
@@ -64,7 +64,7 @@
   os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
 
 class Reduce(object):
-  def __init__(self, crash_script, file_to_reduce):
+  def __init__(self, crash_script, file_to_reduce, core_number):
 crash_script_name, crash_script_ext = os.path.splitext(crash_script)
 file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -78,6 +78,8 @@
 self.expected_output = []
 self.needs_stack_trace = False
 self.creduce_flags = ["--tidy"]
+if core_number > 0:
+  self.creduce_flags = ["--n", str(core_number)]
 
 self.read_clang_args(crash_script, file_to_reduce)
 self.read_expected_output()
@@ -394,6 +396,8 @@
   parser.add_argument('--creduce', dest='creduce', type=str,
   help="The path to the `creduce` executable. "
   "Required if `creduce` is not in PATH environment.")
+  parser.add_argument('--n', dest='core_number', type=int, default=0,
+  help="Number of cores to use.")
   parser.add_argument('-v', '--verbose', action='store_true')
   args = parser.parse_args()
 
@@ -401,11 +405,12 @@
   llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
   creduce_cmd = check_cmd('creduce', None, args.creduce)
   clang_cmd = check_cmd('clang', llvm_bin, args.clang)
+  core_number = args.core_number
 
   crash_script = check_file(args.crash_script[0])
   file_to_reduce = check_file(args.file_to_reduce[0])
 
-  r = Reduce(crash_script, file_to_reduce)
+  r = Reduce(crash_script, file_to_reduce, core_number)
 
   r.simplify_clang_args()
   r.write_interestingness_test()
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits