Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/5574#discussion_r28742045
--- Diff: core/src/main/scala/org/apache/spark/util/Utils.scala ---
@@ -1037,21 +1037,52 @@ private[spark] object Utils extends Logging {
}
/**
- * Convert a Java memory parameter passed to -Xmx (such as 300m or 1g)
to a number of megabytes.
+ * Convert a passed byte string (e.g. 50b, 100k, or 250m) to bytes for
+ * internal use.
+ *
+ * If no suffix is provided, the passed number is assumed to be in bytes.
+ */
+ def byteStringAsBytes(str: String): Long = {
+ JavaUtils.byteStringAsBytes(str)
+ }
+
+ /**
+ * Convert a passed byte string (e.g. 50b, 100k, or 250m) to kibibytes
for
+ * internal use.
+ *
+ * If no suffix is provided, the passed number is assumed to be in
kibibytes.
+ */
+ def byteStringAsKb(str: String): Long = {
+ JavaUtils.byteStringAsKb(str)
+ }
+
+ /**
+ * Convert a passed byte string (e.g. 50b, 100k, or 250m) to mebibytes
for
+ * internal use.
+ *
+ * If no suffix is provided, the passed number is assumed to be in
mebibytes.
+ */
+ def byteStringAsMb(str: String): Long = {
+ JavaUtils.byteStringAsMb(str)
+ }
+
+ /**
+ * Convert a passed byte string (e.g. 50b, 100k, or 250m, 500g) to
gibibytes for
+ * internal use.
+ *
+ * If no suffix is provided, the passed number is assumed to be in
gibibytes.
+ */
+ def byteStringAsGb(str: String): Long = {
+ JavaUtils.byteStringAsGb(str)
+ }
+
+ /**
+ * Convert a Java memory parameter passed to -Xmx (such as 300m or 1g)
to a number of mebibytes.
*/
def memoryStringToMb(str: String): Int = {
- val lower = str.toLowerCase
- if (lower.endsWith("k")) {
- (lower.substring(0, lower.length-1).toLong / 1024).toInt
- } else if (lower.endsWith("m")) {
- lower.substring(0, lower.length-1).toInt
- } else if (lower.endsWith("g")) {
- lower.substring(0, lower.length-1).toInt * 1024
- } else if (lower.endsWith("t")) {
- lower.substring(0, lower.length-1).toInt * 1024 * 1024
- } else {// no suffix, so it's just a number in bytes
- (lower.toLong / 1024 / 1024).toInt
- }
+ // Convert to bytes, rather than directly to MB, because when no units
are specified the unit
+ // is assumed to be bytes
+ (JavaUtils.byteStringAsBytes(str) / 1048576.0).toInt
--- End diff --
I would do `/ 1024 / 1024` so it's clearer
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
