Enable heap prof of stmgr in heron-shell.

heron/executor/src/python/heron_executor.py

@@ -770,7 +770,7 @@ def start_process_monitor(self):
               Log.info("%s exited too many times" % name)
               sys.exit(1)
             time.sleep(self.interval_between_runs)
-            p = self._run_process(name, command)
+            p = self._run_process(name, command, self.shell_env)
             del self.processes_to_monitor[pid]
             self.processes_to_monitor[p.pid] =\
               ProcessInfo(p, name, command, old_process_info.attempts + 1)
@@ -892,6 +892,13 @@ def main():
   # PEX_ROOT shell environment before forking the processes
   shell_env = os.environ.copy()
   shell_env["PEX_ROOT"] = os.path.join(os.path.abspath('.'), ".pex")
+  # Refer to https://gperftools.github.io/gperftools/heapprofile.html
+  # for details of settings of gperftools heap profiler
+  shell_env["HEAPPROFILE"] = "stmgr.hprof"
+  shell_env["HEAP_PROFILE_ALLOCATION_INTERVAL"] = "0"
+  shell_env["HEAP_PROFILE_INUSE_INTERVAL"] = "0"
+  shell_env["HEAPPROFILESIGNAL"] = str(signal.SIGUSR1)
+
 
   # Instantiate the executor, bind it to signal handlers and launch it
   executor = HeronExecutor(sys.argv, shell_env)

heron/shell/src/python/handlers/__init__.py

@@ -8,3 +8,4 @@
 from jstackhandler import JstackHandler
 from memoryhistogramhandler import MemoryHistogramHandler
 from pidhandler import PidHandler
+from stmgrheapprofhandler import StmgrHeapProfHandler

heron/shell/src/python/handlers/stmgrheapprofhandler.py

@@ -0,0 +1,45 @@
+# Copyright 2017 Twitter. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+''' stmgrheapprofhandler.py '''
+import glob
+import json
+import os
+import signal
+import tornado.web
+
+from heron.shell.src.python import utils
+
+class StmgrHeapProfHandler(tornado.web.RequestHandler):
+    """
+    Responsible for getting the process ID for an instance.
+    """
+
+    # pylint: disable=attribute-defined-outside-init
+    @tornado.web.asynchronous
+    def get(self):
+        ''' get method '''
+        self.content_type = 'application/json'
+        stmgr_pid_files = glob.glob('stmgr*.pid')
+        try:
+          pid_file = stmgr_pid_files[0]
+          with open(pid_file, 'r') as f:
+            pid = f.read()
+            os.kill(int(pid), signal.SIGUSR1)
+          self.write('Performing heap profiling on stream manager...')
+          self.finish()
+        except:
+          self.write("Not stream manager found")
+          self.set_status(404)
+          self.finish()

heron/shell/src/python/main.py

@@ -34,6 +34,7 @@
     (r"^/filedata/(.*)", handlers.FileDataHandler),
     (r"^/filestats/(.*)", handlers.FileStatsHandler),
     (r"^/download/(.*)", handlers.DownloadHandler),
+    (r"^/stmgrheapprof", handlers.StmgrHeapProfHandler),
 ])
 
 

heron/stmgr/src/cpp/BUILD

@@ -111,6 +111,7 @@ cc_binary(
         "server/stmgr-main.cpp",
     ],
     copts = [
+        "-Ithird_party",
         "-Iheron",
         "-Iheron/common/src/cpp",
         "-Iheron/statemgrs/src/cpp",
@@ -129,6 +130,7 @@ cc_binary(
         "//heron/common/src/cpp/metrics:metrics-cxx",
         "//heron/statemgrs/src/cpp:statemgrs-cxx",
         "//third_party/yaml-cpp:yaml-cxx",
+        "//third_party/gperftools:profiler-cxx",
     ],
     linkstatic = 1,
 )

heron/stmgr/src/cpp/server/stmgr-main.cpp

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#include <gperftools/heap-profiler.h>
 #include <iostream>
 #include <string>
 #include <vector>
@@ -55,6 +56,8 @@ int main(int argc, char* argv[]) {
   sp_int32 shell_port = atoi(argv[11]);
   sp_string heron_internals_config_filename = argv[12];
 
+  HeapProfilerStart("stmgr");
+
   EventLoopImpl ss;
 
   // Read heron internals config from local file
@@ -82,5 +85,6 @@ int main(int argc, char* argv[]) {
                           high_watermark, low_watermark);
   mgr.Init();
   ss.loop();
+  HeapProfilerStop();
   return 0;
 }

third_party/gperftools/BUILD

@@ -100,6 +100,7 @@ cc_library(
         "lib/libprofiler.a",
     ],
     hdrs = [
+        "include/gperftools/heap-profiler.h",
         "include/gperftools/profiler.h",
     ],
     includes = ["include"],