[NO ISSUE] IOCounter improvements - cache results for short period - split iostat, proc fs impls
Change-Id: I7789171db6b6d7eea3561c24467af63f065f5dc6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2515 Reviewed-by: Murtadha Hubail <mhub...@apache.org> Tested-by: Murtadha Hubail <mhub...@apache.org> Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/94b6da6a Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/94b6da6a Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/94b6da6a Branch: refs/heads/release-0.9.4-pre-rc Commit: 94b6da6ae33749dfdee35e35f32c165693015e3d Parents: 2587f1d Author: Michael Blow <mb...@apache.org> Authored: Thu Mar 22 20:15:37 2018 -0400 Committer: Michael Blow <mb...@apache.org> Committed: Thu Mar 22 22:11:31 2018 -0700 ---------------------------------------------------------------------- .../control/nc/io/profiling/IIOCounter.java | 8 +- .../control/nc/io/profiling/IOCounterCache.java | 41 ++++++ .../nc/io/profiling/IOCounterDefault.java | 6 +- .../nc/io/profiling/IOCounterFactory.java | 9 +- .../nc/io/profiling/IOCounterIoStat.java | 101 +++++++++++++++ .../control/nc/io/profiling/IOCounterLinux.java | 128 ------------------- .../control/nc/io/profiling/IOCounterOSX.java | 6 +- .../control/nc/io/profiling/IOCounterProc.java | 71 ++++++++++ 8 files changed, 231 insertions(+), 139 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java index 3612d8f..a85ca2c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java @@ -22,12 +22,12 @@ package org.apache.hyracks.control.nc.io.profiling; public interface IIOCounter { /** - * @return the number of block reads from the very beginning + * @return the number of block reads from the very beginning, or -1 if not available on this platform */ - public long getReads(); + long getReads(); /** - * @return the number of block writes from the very beginning + * @return the number of block writes from the very beginning, or -1 if not available on this platform */ - public long getWrites(); + long getWrites(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java new file mode 100644 index 0000000..842d82b --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.control.nc.io.profiling; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.apache.hyracks.util.Span; + +abstract class IOCounterCache<T> implements IIOCounter { + private static final long TTL_NANOS = TimeUnit.MILLISECONDS.toNanos(500); + private Span span; + private T info; + + protected synchronized T getInfo() throws IOException { + if (info == null || span.elapsed()) { + span = Span.start(TTL_NANOS, TimeUnit.NANOSECONDS); + info = calculateInfo(); + } + return info; + } + + protected abstract T calculateInfo() throws IOException; +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java index 1f8669d..d5ec9a0 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java @@ -21,14 +21,16 @@ package org.apache.hyracks.control.nc.io.profiling; public class IOCounterDefault implements IIOCounter { + public static final long IO_COUNTER_UNAVAILABLE = -1; + @Override public long getReads() { - return 0; + return IO_COUNTER_UNAVAILABLE; } @Override public long getWrites() { - return 0; + return IO_COUNTER_UNAVAILABLE; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java index 2301ae6..36b310d 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java @@ -33,9 +33,12 @@ public class IOCounterFactory { */ public IIOCounter getIOCounter() { String osName = System.getProperty("os.name").toLowerCase(); - if (osName.indexOf("nix") >= 0 || osName.indexOf("nux") >= 0 || osName.indexOf("aix") >= 0) { - return new IOCounterLinux(); - } else if (osName.indexOf("mac") >= 0) { + if (osName.contains("nix") || osName.contains("nux") || osName.contains("aix")) { + if (IOCounterProc.STATFILE.exists()) { + return new IOCounterProc(); + } + return new IOCounterIoStat(); + } else if (osName.contains("mac")) { return new IOCounterOSX(); } else { return new IOCounterDefault(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java new file mode 100644 index 0000000..560035b --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.control.nc.io.profiling; + +import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.List; +import java.util.StringTokenizer; + +import org.apache.commons.io.IOUtils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class IOCounterIoStat extends IOCounterCache<List<String>> { + private static final Logger LOGGER = LogManager.getLogger(); + private static final String COMMAND = "iostat"; + private static final int PAGE_SIZE = 512; + private long failureCount; + + private long baseReads; + private long baseWrites; + + IOCounterIoStat() { + baseReads = getReads(); + baseWrites = getWrites(); + } + + @Override + public long getReads() { + try { + long reads = extractColumn(4) * PAGE_SIZE; + return reads == 0 ? IO_COUNTER_UNAVAILABLE : reads - baseReads; + } catch (Exception e) { + LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting reads", e); + return IO_COUNTER_UNAVAILABLE; + } + } + + @Override + public long getWrites() { + try { + long writes = extractColumn(5) * PAGE_SIZE; + return writes == 0 ? IO_COUNTER_UNAVAILABLE : writes - baseWrites; + } catch (Exception e) { + LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting writes", e); + return IO_COUNTER_UNAVAILABLE; + } + } + + private long extractColumn(int columnIndex) throws IOException { + boolean device = false; + long ios = 0; + for (String line : getInfo()) { + if (line.contains("Blk_read")) { + device = true; + continue; + } + if (device) { + StringTokenizer tokenizer = new StringTokenizer(line); + int i = 0; + while (tokenizer.hasMoreTokens()) { + String column = tokenizer.nextToken(); + if (i == columnIndex) { + ios += Long.parseLong(column); + break; + } + i++; + } + } + } + return ios; + } + + @Override + protected List<String> calculateInfo() throws IOException { + try (InputStream inputStream = Runtime.getRuntime().exec(COMMAND).getInputStream()) { + return IOUtils.readLines(inputStream, Charset.defaultCharset()); + } + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java deleted file mode 100644 index 3db4c8c..0000000 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.control.nc.io.profiling; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.StringTokenizer; - -public class IOCounterLinux implements IIOCounter { - public static final String COMMAND = "iostat"; - public static final String STATFILE = "/proc/self/io"; - public static final int PAGE_SIZE = 4096; - - private long baseReads = 0; - private long baseWrites = 0; - - public IOCounterLinux() { - baseReads = getReads(); - baseWrites = getWrites(); - } - - @Override - public long getReads() { - try { - long reads = extractRow(4); - return reads; - } catch (IOException e) { - try { - long reads = extractColumn(4) * PAGE_SIZE; - return reads - baseReads; - } catch (IOException e2) { - return 0; - } - } - } - - @Override - public long getWrites() { - try { - long writes = extractRow(5); - long cancelledWrites = extractRow(6); - return (writes - cancelledWrites); - } catch (IOException e) { - try { - long writes = extractColumn(5) * PAGE_SIZE; - return writes - baseWrites; - } catch (IOException e2) { - return 0; - } - } - } - - private long extractColumn(int columnIndex) throws IOException { - BufferedReader reader = exec(COMMAND); - String line = null; - boolean device = false; - long ios = 0; - while ((line = reader.readLine()) != null) { - if (line.contains("Blk_read")) { - device = true; - continue; - } - if (device == true) { - StringTokenizer tokenizer = new StringTokenizer(line); - int i = 0; - while (tokenizer.hasMoreTokens()) { - String column = tokenizer.nextToken(); - if (i == columnIndex) { - ios += Long.parseLong(column); - break; - } - i++; - } - } - } - reader.close(); - return ios; - } - - private long extractRow(int rowIndex) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(STATFILE))); - String line = null; - long ios = 0; - int i = 0; - while ((line = reader.readLine()) != null) { - if (i == rowIndex) { - StringTokenizer tokenizer = new StringTokenizer(line); - int j = 0; - while (tokenizer.hasMoreTokens()) { - String column = tokenizer.nextToken(); - if (j == 1) { - ios = Long.parseLong(column); - break; - } - j++; - } - } - i++; - } - reader.close(); - return ios; - } - - private BufferedReader exec(String command) throws IOException { - Process p = Runtime.getRuntime().exec(command); - return new BufferedReader(new InputStreamReader(p.getInputStream())); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java index 729157b..a48d55d 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java @@ -19,16 +19,18 @@ package org.apache.hyracks.control.nc.io.profiling; +import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE; + public class IOCounterOSX implements IIOCounter { @Override public long getReads() { - return 0; + return IO_COUNTER_UNAVAILABLE; } @Override public long getWrites() { - return 0; + return IO_COUNTER_UNAVAILABLE; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java new file mode 100644 index 0000000..8882271 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.control.nc.io.profiling; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class IOCounterProc extends IOCounterCache<List<String>> { + private static final Logger LOGGER = LogManager.getLogger(); + @SuppressWarnings("squid:S1075") // hardcoded URI + public static final File STATFILE = new File("/proc/self/io"); + private long failureCount; + + @Override + public long getReads() { + try { + return extractRow(getInfo(), 4); + } catch (Exception e) { + LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting reads", e); + return IOCounterDefault.IO_COUNTER_UNAVAILABLE; + } + } + + @Override + public long getWrites() { + try { + List<String> rows = getInfo(); + long writes = extractRow(rows, 5); + long cancelledWrites = extractRow(rows, 6); + return writes - cancelledWrites; + } catch (Exception e) { + LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting writes", e); + return IOCounterDefault.IO_COUNTER_UNAVAILABLE; + } + } + + private long extractRow(List<String> rows, int rowIndex) { + return Long.parseLong(StringUtils.split(rows.get(rowIndex), ' ')[1]); + } + + @Override + protected List<String> calculateInfo() throws IOException { + return FileUtils.readLines(STATFILE, Charset.defaultCharset()); + } + +}
