From 137c6dca075ca598694e1a23a130fab0b40ff109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luca=20Kov=C3=A1cs?= Date: Thu, 20 Oct 2022 12:50:12 +0200 Subject: [PATCH 01/10] HBASE-27406 Make /prometheus endpoint accessible from HBase UI (#4833) Signed-off-by: Andor Molnar Signed-off-by: Balazs Meszaros (cherry picked from commit dffc8e0fbefda603e3c0557bc2ceac600152675e) --- .../src/main/resources/hbase-webapps/rest/rest.jsp | 12 +++++++++++- .../hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon | 12 +++++++++++- .../hbase/tmpl/regionserver/RSStatusTmpl.jamon | 12 +++++++++++- .../main/resources/hbase-webapps/master/header.jsp | 12 +++++++++++- .../resources/hbase-webapps/regionserver/header.jsp | 12 +++++++++++- .../main/resources/hbase-webapps/thrift/thrift.jsp | 12 +++++++++++- 6 files changed, 66 insertions(+), 6 deletions(-) diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp index 3deb2bbc7357..df8f0838d6cc 100644 --- a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp @@ -58,7 +58,17 @@ String listenPort = conf.get("hbase.rest.port", "8080");
  • Home
  • Local logs
  • Log Level
  • -
  • Metrics Dump
  • +
  • Profiler
  • <% if (HBaseConfiguration.isShowConfInServlet()) { %>
  • HBase Configuration
  • diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 863d7e039a40..c586b2226646 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -162,7 +162,17 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
  • Local Logs
  • Log Level
  • Debug Dump
  • -
  • Metrics Dump
  • +
  • Profiler
  • <%if HBaseConfiguration.isShowConfInServlet()%>
  • HBase Configuration
  • diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 2068c7607990..23330096cf39 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -114,7 +114,17 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
  • Operation Details
  • Log Level
  • Debug Dump
  • -
  • Metrics Dump
  • +
  • Profiler
  • <%if HBaseConfiguration.isShowConfInServlet()%>
  • HBase Configuration
  • diff --git a/hbase-server/src/main/resources/hbase-webapps/master/header.jsp b/hbase-server/src/main/resources/hbase-webapps/master/header.jsp index 7f4fa55f59da..54ffd4e03415 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/header.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/header.jsp @@ -69,7 +69,17 @@
  • Local Logs
  • Log Level
  • Debug Dump
  • -
  • Metrics Dump
  • +
  • Profiler
  • <% if (HBaseConfiguration.isShowConfInServlet()) { %>
  • HBase Configuration
  • diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/header.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/header.jsp index edbecc424c62..1c5318b19b94 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/header.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/header.jsp @@ -56,7 +56,17 @@
  • Operation Details
  • Log Level
  • Debug Dump
  • -
  • Metrics Dump
  • +
  • Profiler
  • <% if (HBaseConfiguration.isShowConfInServlet()) { %>
  • HBase Configuration
  • diff --git a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp index cb22007bea6f..dea591230995 100644 --- a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp +++ b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp @@ -66,7 +66,17 @@ String compact = conf.get("hbase.regionserver.thrift.compact", "false");
  • Home
  • Local logs
  • Log Level
  • -
  • Metrics Dump
  • +
  • Profiler
  • <% if (HBaseConfiguration.isShowConfInServlet()) { %>
  • HBase Configuration
  • From 36bd555b1db3dac0069e3a57b53013005d42eec5 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sun, 14 Jan 2024 11:59:26 +0530 Subject: [PATCH 02/10] HBASE-27814 Add support for dump and process metrics servlet in REST InfoServer (#5215) Other changes: - Ensure info server stops during stop() - Extract header and footer. This would fix the log level page layout for rest web UI (See HBASE-20693) - Add hostname in the landing page instead of just port similar to other web UIs Signed-off-by: Nick Dimiduk (cherry picked from commit a683fcfbe8c5c84e58fa8670f4414f9d01ff43ed) --- .../hadoop/hbase/rest/RESTDumpServlet.java | 80 ++++++++ .../apache/hadoop/hbase/rest/RESTServer.java | 39 +++- .../resources/hbase-webapps/rest/footer.jsp | 32 +++ .../resources/hbase-webapps/rest/header.jsp | 74 +++++++ .../hbase-webapps/rest/processRest.jsp | 184 ++++++++++++++++++ .../resources/hbase-webapps/rest/rest.jsp | 80 ++------ 6 files changed, 422 insertions(+), 67 deletions(-) create mode 100644 hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java create mode 100644 hbase-rest/src/main/resources/hbase-webapps/rest/footer.jsp create mode 100644 hbase-rest/src/main/resources/hbase-webapps/rest/header.jsp create mode 100644 hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java new file mode 100644 index 000000000000..8bb306f7829a --- /dev/null +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTDumpServlet.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.Date; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.http.HttpServer; +import org.apache.hadoop.hbase.monitoring.StateDumpServlet; +import org.apache.hadoop.hbase.util.LogMonitoring; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class RESTDumpServlet extends StateDumpServlet { + private static final long serialVersionUID = 1L; + private static final String LINE = "==========================================================="; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), request, response)) { + return; + } + + RESTServer restServer = (RESTServer) getServletContext().getAttribute(RESTServer.REST_SERVER); + assert restServer != null : "No REST Server in context!"; + + response.setContentType("text/plain"); + OutputStream os = response.getOutputStream(); + try (PrintWriter out = new PrintWriter(os)) { + + out.println("REST Server status for " + restServer.getServerName() + " as of " + new Date()); + + out.println("\n\nVersion Info:"); + out.println(LINE); + dumpVersionInfo(out); + + out.println("\n\nStacks:"); + out.println(LINE); + out.flush(); + PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8"); + Threads.printThreadInfo(ps, ""); + ps.flush(); + + out.println("\n\nREST Server configuration:"); + out.println(LINE); + Configuration conf = restServer.conf; + out.flush(); + conf.writeXml(os); + os.flush(); + + out.println("\n\nLogs"); + out.println(LINE); + long tailKb = getTailKbParam(request); + LogMonitoring.dumpTailOfLogs(out, tailKb); + + out.flush(); + } + } +} diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index 5796cd066f9d..dea6f2909875 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hbase.http.HttpServerUtil.PATH_SPEC_ANY; import java.lang.management.ManagementFactory; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -31,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.http.HttpServerUtil; import org.apache.hadoop.hbase.http.InfoServer; import org.apache.hadoop.hbase.log.HBaseMarkers; @@ -83,6 +85,7 @@ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) public class RESTServer implements Constants { static Logger LOG = LoggerFactory.getLogger("RESTServer"); + public static final String REST_SERVER = "rest"; static final String REST_CSRF_ENABLED_KEY = "hbase.rest.csrf.enabled"; static final boolean REST_CSRF_ENABLED_DEFAULT = false; @@ -110,6 +113,7 @@ public class RESTServer implements Constants { private final UserProvider userProvider; private Server server; private InfoServer infoServer; + private ServerName serverName; public RESTServer(Configuration conf) { RESTServer.conf = conf; @@ -143,8 +147,7 @@ void addCSRFFilter(ServletContextHandler ctxHandler, Configuration conf) { loginServerPrincipal(UserProvider userProvider, Configuration conf) throws Exception { Class containerClass = ServletContainer.class; if (userProvider.isHadoopSecurityEnabled() && userProvider.isHBaseSecurityEnabled()) { - String machineName = Strings.domainNamePointerToHostName(DNS.getDefaultHost( - conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); + String machineName = getHostName(conf); String keytabFilename = conf.get(REST_KEYTAB_FILE); Preconditions.checkArgument(keytabFilename != null && !keytabFilename.isEmpty(), REST_KEYTAB_FILE + " should be set if security is enabled"); @@ -385,9 +388,14 @@ public synchronized void run() throws Exception { // Put up info server. int port = conf.getInt("hbase.rest.info.port", 8085); if (port >= 0) { - conf.setLong("startcode", EnvironmentEdgeManager.currentTime()); - String a = conf.get("hbase.rest.info.bindAddress", "0.0.0.0"); - this.infoServer = new InfoServer("rest", a, port, false, conf); + final long startCode = EnvironmentEdgeManager.currentTime(); + conf.setLong("startcode", startCode); + this.serverName = ServerName.valueOf(getHostName(conf), servicePort, startCode); + + String addr = conf.get("hbase.rest.info.bindAddress", "0.0.0.0"); + this.infoServer = new InfoServer(REST_SERVER, addr, port, false, conf); + this.infoServer.addPrivilegedServlet("dump", "/dump", RESTDumpServlet.class); + this.infoServer.setAttribute(REST_SERVER, this); this.infoServer.setAttribute("hbase.conf", conf); this.infoServer.start(); } @@ -395,6 +403,11 @@ public synchronized void run() throws Exception { server.start(); } + private static String getHostName(Configuration conf) throws UnknownHostException { + return Strings.domainNamePointerToHostName(DNS.getDefaultHost( + conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); + } + public synchronized void join() throws Exception { if (server == null) { throw new IllegalStateException("Server is not running"); @@ -402,7 +415,19 @@ public synchronized void join() throws Exception { server.join(); } + private void stopInfoServer() { + if (this.infoServer != null) { + LOG.info("Stop info server"); + try { + this.infoServer.stop(); + } catch (Exception e) { + LOG.error("Failed to stop infoServer", e); + } + } + } + public synchronized void stop() throws Exception { + stopInfoServer(); if (server == null) { throw new IllegalStateException("Server is not running"); } @@ -426,6 +451,10 @@ public synchronized int getInfoPort() { return infoServer.getPort(); } + public ServerName getServerName() { + return serverName; + } + public Configuration getConf() { return conf; } diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/footer.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/footer.jsp new file mode 100644 index 000000000000..a642ac36eff7 --- /dev/null +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/footer.jsp @@ -0,0 +1,32 @@ +<%-- +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--%> + + + + + + + + diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/header.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/header.jsp new file mode 100644 index 000000000000..67f7656de592 --- /dev/null +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/header.jsp @@ -0,0 +1,74 @@ +<%-- +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="org.apache.hadoop.hbase.HBaseConfiguration"%> + + + + + + + <%= request.getParameter("pageTitle")%> + + + + + + + + + + + diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp new file mode 100644 index 000000000000..2b2d35fbfb3f --- /dev/null +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/processRest.jsp @@ -0,0 +1,184 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.Date" + import="java.util.List" + import="javax.management.ObjectName" + import="java.lang.management.ManagementFactory" + import="java.lang.management.MemoryPoolMXBean" + import="java.lang.management.RuntimeMXBean" + import="java.lang.management.GarbageCollectorMXBean" + import="org.apache.hadoop.hbase.util.JSONMetricUtil" + import="org.apache.hadoop.hbase.procedure2.util.StringUtils" + import="org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix" +%> + +<% +RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); +ObjectName jvmMetrics = new ObjectName("Hadoop:service=HBase,name=JvmMetrics"); + +// There is always two of GC collectors +List gcBeans = JSONMetricUtil.getGcCollectorBeans(); +GarbageCollectorMXBean collector1 = null; +GarbageCollectorMXBean collector2 = null; +try { +collector1 = gcBeans.get(0); +collector2 = gcBeans.get(1); +} catch(IndexOutOfBoundsException e) {} +List mPools = JSONMetricUtil.getMemoryPools(); +pageContext.setAttribute("pageTitle", "Process info for PID: " + JSONMetricUtil.getProcessPID()); +%> + + + + + +
    +
    + +
    + + + + + + + + + + + + + + +
    StartedUptimePIDOwner
    <%= new Date(runtimeBean.getStartTime()) %><%= StringUtils.humanTimeDiff(runtimeBean.getUptime()) %><%= JSONMetricUtil.getProcessPID() %><%= runtimeBean.getSystemProperties().get("user.name") %>
    +
    +
    +
    + +
    + + + + + + + + + + + + + + + + + +
    ThreadsNewThreadsRunableThreadsBlockedThreadsWaitingThreadsTimeWaitingThreadsTerminated
    <%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsNew") %><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsRunnable")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsBlocked")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsWaiting")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsTimedWaiting")%><%= JSONMetricUtil.getValueFromMBean(jvmMetrics, "ThreadsTerminated")%>
    +
    +
    +
    + +
    + <% if (gcBeans.size() == 2) { %> +
    + +
    +
    + + + + + + + + + + + +
    Collection CountCollection TimeLast duration
    <%= collector1.getCollectionCount() %> <%= StringUtils.humanTimeDiff(collector1.getCollectionTime()) %> <%= StringUtils.humanTimeDiff(JSONMetricUtil.getLastGcDuration( + collector1.getObjectName())) %>
    +
    +
    + + + + + + + + + + + +
    Collection CountCollection TimeLast duration
    <%= collector2.getCollectionCount() %> <%= StringUtils.humanTimeDiff(collector2.getCollectionTime()) %> <%= StringUtils.humanTimeDiff(JSONMetricUtil.getLastGcDuration( + collector2.getObjectName())) %>
    +
    +
    +
    + <%} else { %> +

    Can not display GC Collector stats.

    + <%} %> + Total GC Collection time: <%= StringUtils.humanTimeDiff(collector1.getCollectionTime() + + collector2.getCollectionTime())%> +
    +<% for(MemoryPoolMXBean mp:mPools) { +if(mp.getName().contains("Cache")) continue;%> +
    +
    + +
    + + + + + + + + + + + + + + + + +
    CommitedInitMaxUsedUtilization [%]
    <%= TraditionalBinaryPrefix.long2String(mp.getUsage().getCommitted(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getInit(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getMax(), "B", 1) %><%= TraditionalBinaryPrefix.long2String(mp.getUsage().getUsed(), "B", 1) %><%= JSONMetricUtil.calcPercentage(mp.getUsage().getUsed(), + mp.getUsage().getCommitted()) %>
    +
    +<% } %> + + diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp index df8f0838d6cc..ce6725f283a7 100644 --- a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp @@ -18,70 +18,29 @@ */ --%> <%@ page contentType="text/html;charset=UTF-8" - import="org.apache.hadoop.conf.Configuration" - import="org.apache.hadoop.hbase.HBaseConfiguration" - import="org.apache.hadoop.hbase.rest.model.VersionModel" - import="org.apache.hadoop.hbase.util.VersionInfo" - import="java.util.Date"%> + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.rest.RESTServer" + import="org.apache.hadoop.hbase.rest.model.VersionModel" + import="org.apache.hadoop.hbase.util.VersionInfo" + import="java.util.Date"%> + <% -Configuration conf = (Configuration)getServletContext().getAttribute("hbase.conf"); -long startcode = conf.getLong("startcode", System.currentTimeMillis()); -String listenPort = conf.get("hbase.rest.port", "8080"); -%> - - - - - - HBase REST Server: <%= listenPort %> - - + Configuration conf = (Configuration) getServletContext().getAttribute("hbase.conf"); + long startcode = conf.getLong("startcode", System.currentTimeMillis()); - - - - + final RESTServer restServer = (RESTServer) getServletContext().getAttribute(RESTServer.REST_SERVER); + final String hostName = restServer.getServerName().getHostname(); + pageContext.setAttribute("pageTitle", "HBase REST Server" + hostName); +%> - - + + +
    @@ -124,9 +83,6 @@ String listenPort = conf.get("hbase.rest.port", "8080");
    - - - - - + + From 6a4dbea79465d02390545d5c3f9df0b50085e24a Mon Sep 17 00:00:00 2001 From: Beata Sudi Date: Mon, 9 Dec 2019 10:24:19 +0100 Subject: [PATCH 03/10] HBASE-18382 add transport type info into Thrift UI (#880) Signed-off-by: Wellington Chevreuil Signed-off-by: Bharath Vissapragada Signed-off-by: Viraj Jasani (cherry picked from commit 82e155eb26cddd642e151d9276e2f4f281b4df88) --- .../resources/hbase-webapps/thrift/thrift.jsp | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp index dea591230995..6b308fc2a2dc 100644 --- a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp +++ b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp @@ -32,9 +32,14 @@ String serverType = (String)getServletContext().getAttribute("hbase.thrift.serve long startcode = conf.getLong("startcode", System.currentTimeMillis()); String listenPort = conf.get("hbase.regionserver.thrift.port", "9090"); ImplType implType = ImplType.getServerImpl(conf); -String framed = implType.isAlwaysFramed() - ? "true" : conf.get("hbase.regionserver.thrift.framed", "false"); -String compact = conf.get("hbase.regionserver.thrift.compact", "false"); + +String transport = + (implType.isAlwaysFramed() || + conf.getBoolean("hbase.regionserver.thrift.framed", false)) ? "Framed" : "Standard"; +String protocol = + conf.getBoolean("hbase.regionserver.thrift.compact", false) ? "Compact" : "Binary"; +String qop = conf.get("hbase.thrift.security.qop", "None"); + %> @@ -54,12 +59,15 @@ String compact = conf.get("hbase.regionserver.thrift.compact", "false"); +
    + + + + + + + + + + + + <% - for (Map.Entry rdEntry : regDistribution.entrySet()) { - ServerName addr = rdEntry.getKey(); - String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; + numRegionsRendered = 0; + for (Map.Entry hriEntry : entryList) { + RegionInfo regionInfo = hriEntry.getKey(); + ServerName addr = regionsToServer.get(regionInfo); + RegionMetrics load = hriEntry.getValue(); + long compactingCells = 0; + long compactedCells = 0; + String compactionProgress = ""; + if (load != null) { + compactingCells = load.getCompactingCellCount(); + compactedCells = load.getCompactedCellCount(); + if (compactingCells > 0) { + compactionProgress = String.format("%.2f", 100 * ((float) + compactedCells / compactingCells)) + "%"; + } + } + + if (numRegionsRendered < numRegionsToRender) { + numRegionsRendered++; %> - - - <% - if (withReplica) { - %> - - <% - } - %> + + <%= buildRegionDeployedServerTag(regionInfo, master, regionsToServer) %> + + + + <% } %> + <% } %>
    Name(<%= String.format("%,1d", regions.size())%>)Region ServerNum. Compacting Cells
    (<%= String.format("%,1d", totalCompactingCells)%>)
    Num. Compacted Cells
    (<%= String.format("%,1d", totalCompactedCells)%>)
    Remaining Cells
    (<%= String.format("%,1d", totalCompactingCells-totalCompactedCells)%>)
    Compaction Progress
    (<%= totalCompactionProgress %>)
    <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %><%= rdEntry.getValue()%><%= primaryRegDistribution.get(addr)%><%= escapeXml(Bytes.toStringBinary(regionInfo.getRegionName())) %><%= String.format("%,1d", compactingCells)%><%= String.format("%,1d", compactedCells)%><%= String.format("%,1d", compactingCells - compactedCells)%><%= compactionProgress%>
    - <% } + <%= moreRegionsToRender(numRegionsRendered, numRegions, fqtn) %> +
    + + + +

    Regions by Region Server

    + + + + + + + + + <% + for (Map.Entry rdEntry : regDistribution.entrySet()) { + ServerName addr = rdEntry.getKey(); + String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + + master.getRegionServerInfoPort(addr) + "/rs-status"; + %> + + + + + + <% } %> + +
    Region ServerRegion CountPrimary Region Count
    <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + + ":" + master.getRegionServerInfoPort(addr) %><%= rdEntry.getValue()%><%= primaryRegDistribution.get(addr) == null ? 0 : primaryRegDistribution.get(addr)%>
    + +<% } } catch(Exception ex) { %> Unknown Issue with Regions
    @@ -1246,41 +1210,6 @@ <% } %>
    -<% } -} catch(TableNotFoundException e) { %> -
    -
    - -
    -


    -

    Go Back -

    <% -} catch(IllegalArgumentException e) { %> -
    -
    - -
    -


    -

    Go Back -

    <% - } -} -else { // handle the case for fqtn is null or master is not initialized with error message + redirect -%> -
    -
    - -
    -


    - -

    -<% } %> @@ -1288,78 +1217,90 @@ else { // handle the case for fqtn is null or master is not initialized with err From b8d5f988d7816217a28448b80b43130161f66e53 Mon Sep 17 00:00:00 2001 From: Akshay Sudheer <74921542+AkshayTSudheer@users.noreply.github.com> Date: Wed, 3 Mar 2021 09:38:55 +0530 Subject: [PATCH 06/10] HBASE-25402 Sorting order by start key or end key is not considering empty start key/end key (#2955) Signed-off-by: Duo Zhang Signed-off-by: Pankaj Kumar (cherry picked from commit 157200ef8375a91905efb3589393f2ef2b58b970) --- .../hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon | 7 ++++++- .../src/main/resources/hbase-webapps/master/table.jsp | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 23330096cf39..d837b8504623 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -299,7 +299,12 @@ $(document).ready(function() type: "numeric" }); - $("#baseStatsTable").tablesorter(); + $("#baseStatsTable").tablesorter({ + headers: { + 1: {empty: 'emptyMin'}, + 2: {empty: 'emptyMax'} + } + }); $("#requestStatsTable").tablesorter({ headers: { 1: {sorter: 'separator'}, diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index d234b0f5b480..a639f18ed1b4 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -1263,7 +1263,9 @@ $(document).ready(function() 3: {sorter: 'separator'}, 4: {sorter: 'filesize'}, 5: {sorter: 'separator'}, - 6: {sorter: 'filesize'} + 6: {sorter: 'filesize'}, + 7: {empty: 'emptyMin'}, + 8: {empty: 'emptyMax'} } }); $("#metaTableBaseStatsTable").tablesorter({ @@ -1272,7 +1274,9 @@ $(document).ready(function() 3: {sorter: 'separator'}, 4: {sorter: 'filesize'}, 5: {sorter: 'separator'}, - 6: {sorter: 'filesize'} + 6: {sorter: 'filesize'}, + 7: {empty: 'emptyMin'}, + 8: {empty: 'emptyMax'} } }); $("#tableLocalityStatsTable").tablesorter({ From 0b0ac60f5f57cae87ce043b2951e61915ea2431e Mon Sep 17 00:00:00 2001 From: SiCheng-Zheng <643463623@qq.com> Date: Tue, 4 Oct 2022 23:10:25 +0800 Subject: [PATCH 07/10] HBASE-27309 Add major compact table or region operation on master web table page (#4793) Co-authored-by: zhengsicheng Signed-off-by: Duo Zhang (cherry picked from commit eb6b2745c550905e92dcfee34e558a0901944da5) --- .../resources/hbase-webapps/master/table.jsp | 159 +++++++++++------- 1 file changed, 95 insertions(+), 64 deletions(-) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index a639f18ed1b4..6dd2124f8ad6 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -232,7 +232,7 @@ <% return; } %> -<% // table split/compact/merge actions +<% // table split/major compact/compact/merge actions if ( !readOnly && action != null ) { %>
    @@ -248,6 +248,20 @@ admin.split(TableName.valueOf(fqtn)); } %> Split request accepted. <% + } else if (action.equals("major compact")) { + if (key != null && key.length() > 0) { + List regions = admin.getRegions(TableName.valueOf(fqtn)).get(); + byte[] row = Bytes.toBytes(key); + + for (RegionInfo region : regions) { + if (region.containsRow(row)) { + admin.majorCompactRegion(region.getRegionName()); + } + } + } else { + admin.majorCompact(TableName.valueOf(fqtn)); + } +%> major Compact request accepted. <% } else if (action.equals("compact")) { if (key != null && key.length() > 0) { List regions = admin.getRegions(TableName.valueOf(fqtn)).get(); @@ -1146,69 +1160,86 @@ - <% if (!readOnly) { %> -


    - Actions: -

    -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - - This action will force a compaction of all regions of the table, or, - if a key is supplied, only the region containing the - given key. -
    - - - - - This action will force a split of all eligible - regions of the table, or, if a key is supplied, only the region containing the - given key. An eligible region is one that does not contain any references to - other regions. Split requests for noneligible regions will be ignored. -
    - - - - - - This action will merge two regions of the table, Merge requests for - noneligible regions will be ignored. -
    -
    -

    - <% } %> -
    +<% if (!readOnly) { %> +


    +Actions: +

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + + + This action will force a major compaction of all regions of the table, or, + if a key is supplied, only the region major containing the + given key. +
    + + + + + This action will force a compaction of all regions of the table, or, + if a key is supplied, only the region containing the + given key. +
    + + + + + This action will force a split of all eligible + regions of the table, or, if a key is supplied, only the region containing the + given key. An eligible region is one that does not contain any references to + other regions. Split requests for noneligible regions will be ignored. +
    + + + + + + This action will merge two regions of the table, Merge requests for + noneligible regions will be ignored. +
    +
    +

    +<% } %> +
    From 8b14486d14436374c0bc833027f784c4f6bd8902 Mon Sep 17 00:00:00 2001 From: Peng Lu Date: Thu, 5 Sep 2024 20:07:36 +0800 Subject: [PATCH 08/10] HBASE-28778 NPE may occur when opening master-status or table.jsp or procedure.jsp while Master is initializing (#6152) Signed-off-by: Duo Zhang (cherry picked from commit 3caaf2d739106b56ab94a0561e730ff35802610d) --- .../resources/hbase-webapps/master/table.jsp | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 6dd2124f8ad6..d03201b78ed0 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -147,11 +147,32 @@ return ""; } %> + + + + + <% final String ZEROMB = "0 MB"; HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); Configuration conf = master.getConfiguration(); String fqtn = request.getParameter("name"); + // handle the case for fqtn is null or master is not initialized with error message + redirect + if (fqtn == null || !master.isInitialized()) { +%> +
    +
    + +
    +


    + +

    +<% return; + } %> + +<% final String escaped_fqtn = StringEscapeUtils.escapeHtml4(fqtn); Table table = master.getConnection().getTable(TableName.valueOf(fqtn)); boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); @@ -200,24 +221,6 @@ final MetaBrowser metaBrowser = new MetaBrowser(connection, request); %> - - - - -<% // handle the case for fqtn is null or master is not initialized with error message + redirect - if (fqtn == null || ! master.isInitialized()) { %> -
    -
    - -
    -


    - -

    -<% return; - } %> - <% // unknow table if (! admin.tableExists(TableName.valueOf(fqtn)).get()) { %>
    From ebdeef2cf5807a17da79918c299a4fbd0f6259c8 Mon Sep 17 00:00:00 2001 From: haosen chen <99318736+haosenchen@users.noreply.github.com> Date: Sat, 13 Jan 2024 17:42:57 +0800 Subject: [PATCH 09/10] HBASE-28305 Add "Uncompressed StoreFileSize" column to the table.jsp (#5620) Co-authored-by: Haosen Chen Signed-off-by: Duo Zhang (cherry picked from commit e3a0174e20542e661f787a874870b629a274daf5) --- .../resources/hbase-webapps/master/table.jsp | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index d03201b78ed0..ccadcc9d7545 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -317,6 +317,7 @@ Region Server ReadRequests WriteRequests + Uncompressed StoreFileSize StorefileSize Num.Storefiles MemSize @@ -340,6 +341,7 @@ String hostAndPort = ""; String readReq = "N/A"; String writeReq = "N/A"; + String fileSizeUncompressed = ZEROMB; String fileSize = ZEROMB; String fileCount = "N/A"; String memSize = ZEROMB; @@ -357,6 +359,10 @@ if (rSize > 0) { fileSize = StringUtils.byteDesc((long) rSize); } + double rSizeUncompressed = load.getUncompressedStoreFileSize().get(Size.Unit.BYTE); + if (rSizeUncompressed > 0) { + fileSizeUncompressed = StringUtils.byteDesc((long) rSizeUncompressed); + } fileCount = String.format("%,1d", load.getStoreFileCount()); double mSize = load.getMemStoreSize().get(Size.Unit.BYTE); if (mSize > 0) { @@ -371,6 +377,7 @@ <%= StringEscapeUtils.escapeHtml4(hostAndPort) %> <%= readReq%> <%= writeReq%> + <%= fileSizeUncompressed%> <%= fileSize%> <%= fileCount%> <%= memSize%> @@ -833,6 +840,7 @@ <% long totalReadReq = 0; long totalWriteReq = 0; + long totalSizeUncompressed = 0; long totalSize = 0; long totalStoreFileCount = 0; long totalMemSize = 0; @@ -843,6 +851,7 @@ long totalBlocksLocalWithSsdWeight = 0; String totalCompactionProgress = ""; String totalMemSizeStr = ZEROMB; + String totalSizeUncompressedStr = ZEROMB; String totalSizeStr = ZEROMB; String totalLocality = ""; String totalLocalityForSsd = ""; @@ -865,6 +874,7 @@ if (regionMetrics != null) { totalReadReq += regionMetrics.getReadRequestCount(); totalWriteReq += regionMetrics.getWriteRequestCount(); + totalSizeUncompressed += regionMetrics.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); totalSize += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE); totalStoreFileCount += regionMetrics.getStoreFileCount(); totalMemSize += regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE); @@ -890,6 +900,9 @@ if (totalSize > 0) { totalSizeStr = StringUtils.byteDesc(totalSize*1024l*1024); } + if (totalSizeUncompressed > 0){ + totalSizeUncompressedStr = StringUtils.byteDesc(totalSizeUncompressed*1024l*1024); + } if (totalMemSize > 0) { totalMemSizeStr = StringUtils.byteDesc(totalMemSize*1024l*1024); } @@ -920,6 +933,7 @@ Region Server ReadRequests
    (<%= String.format("%,1d", totalReadReq)%>) WriteRequests
    (<%= String.format("%,1d", totalWriteReq)%>) + Uncompressed StoreFileSize
    (<%= totalSizeUncompressedStr %>) StorefileSize
    (<%= totalSizeStr %>) Num.Storefiles
    (<%= String.format("%,1d", totalStoreFileCount)%>) MemSize
    (<%= totalMemSizeStr %>) @@ -944,6 +958,7 @@ RegionMetrics load = hriEntry.getValue(); String readReq = "N/A"; String writeReq = "N/A"; + String regionSizeUncompressed = ZEROMB; String regionSize = ZEROMB; String fileCount = "N/A"; String memSize = ZEROMB; @@ -951,6 +966,10 @@ if (load != null) { readReq = String.format("%,1d", load.getReadRequestCount()); writeReq = String.format("%,1d", load.getWriteRequestCount()); + double rSizeUncompressed = load.getUncompressedStoreFileSize().get(Size.Unit.BYTE); + if (rSizeUncompressed > 0) { + regionSizeUncompressed = StringUtils.byteDesc((long)rSizeUncompressed); + } double rSize = load.getStoreFileSize().get(Size.Unit.BYTE); if (rSize > 0) { regionSize = StringUtils.byteDesc((long)rSize); @@ -987,6 +1006,7 @@ <%= buildRegionDeployedServerTag(regionInfo, master, regionsToServer) %> <%= readReq%> <%= writeReq%> + <%= regionSizeUncompressed%> <%= regionSize%> <%= fileCount%> <%= memSize%> From a9d154d856687c590ba504592aaf91a3eb9e29e2 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Wed, 18 Apr 2018 23:25:42 +0530 Subject: [PATCH 10/10] HBASE-20452 Master UI: Table merge button should validate required fields before submit Signed-off-by: tedyu (cherry picked from commit 6ce1136ebae75abc2a1d8d35e1963546b8e2d014) --- .../src/main/resources/hbase-webapps/master/table.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index ccadcc9d7545..6ae98b6985e5 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -1249,8 +1249,8 @@ Actions: - - + + This action will merge two regions of the table, Merge requests for