Reliable list_running_servers(runtime_dir) design

notebook/__init__.py

@@ -1,6 +1,9 @@
 """The Jupyter HTML Notebook"""
 
 import os
+
+JUPYTER_NOTEBOOK_TAG = "JupyterNotebook"
+
 # Packagers: modify this line if you store the notebook static files elsewhere
 DEFAULT_STATIC_FILES_PATH = os.path.join(os.path.dirname(__file__), "static")
 

notebook/base/handlers.py

@@ -34,6 +34,7 @@
 from notebook.i18n import combine_translations
 from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape, urldecode_unix_socket_path
 from notebook.services.security import csp_report_uri
+from notebook import JUPYTER_NOTEBOOK_TAG
 
 #-----------------------------------------------------------------------------
 # Top-level handlers
@@ -849,7 +850,7 @@ class APIVersionHandler(APIHandler):
 
     def get(self):
         # not authenticated, so give as few info as possible
-        self.finish(json.dumps({"version":notebook.__version__}))
+        self.finish(json.dumps({"version": notebook.__version__, "module": JUPYTER_NOTEBOOK_TAG}))
 
 
 class TrailingSlashHandler(web.RequestHandler):

notebook/jstest.py

@@ -31,6 +31,8 @@
 from ipython_genutils.tempdir import TemporaryDirectory
 
 from subprocess import TimeoutExpired
+from notebook.utils import available_port
+
 def popen_wait(p, timeout):
     return p.wait(timeout)
 
@@ -291,11 +293,23 @@ def will_run(self):
         should_run = all(have[a] for a in self.requirements + [self.engine])
         return should_run
 
+    @property
+    def file_id(self):
+        if self.server_port == 0:
+            port_env = os.environ.get('JUPYTER_PORT') or '0'
+            port_env =int(port_env) if port_env.isdigit() else 0
+            if port_env > 0:
+                self.server_port = port_env
+            else:
+                self.server_port = available_port()
+        return str(self.server_port)
+
     def _init_server(self):
         "Start the notebook server in a separate process"
         self.server_command = command = [sys.executable,
             '-m', 'notebook',
             '--no-browser',
+            '--port=%s' % self.file_id,
             '--notebook-dir', self.nbdir.name,
             '--NotebookApp.token=',
             '--NotebookApp.base_url=%s' % self.base_url,
@@ -307,6 +321,7 @@ def _init_server(self):
         self.stream_capturer = c = StreamCapturer()
         c.start()
         env = os.environ.copy()
+        env['PYTHONPATH']= ':'.join(sys.path)
         env.update(self.env)
         self.server = subprocess.Popen(command,
             stdout = c.writefd,
@@ -317,7 +332,7 @@ def _init_server(self):
         with patch.dict('os.environ', {'HOME': self.home.name}):
             runtime_dir = jupyter_runtime_dir()
         self.server_info_file = os.path.join(runtime_dir,
-            'nbserver-%i.json' % self.server.pid
+            'nbserver-%s.json' % self.file_id
         )
         self._wait_for_server()
 

notebook/notebookapp.py

@@ -71,6 +71,7 @@
     from tornado.netutil import bind_unix_socket
 
 from notebook import (
+    JUPYTER_NOTEBOOK_TAG,
     DEFAULT_NOTEBOOK_PORT,
     DEFAULT_STATIC_FILES_PATH,
     DEFAULT_TEMPLATE_PATH_LIST,
@@ -106,6 +107,7 @@
     Any, Dict, Unicode, Integer, List, Bool, Bytes, Instance,
     TraitError, Type, Float, observe, default, validate
 )
+from traitlets.config.configurable import MultipleInstanceError
 from ipython_genutils import py3compat
 from jupyter_core.paths import jupyter_runtime_dir, jupyter_path
 from notebook._sysinfo import get_sys_info
@@ -429,23 +431,12 @@ def start(self):
         self.log.info("Wrote hashed password to %s" % self.config_file)
 
 
-def shutdown_server(server_info, timeout=5, log=None):
-    """Shutdown a notebook server in a separate process.
-
-    *server_info* should be a dictionary as produced by list_running_servers().
-
-    Will first try to request shutdown using /api/shutdown .
-    On Unix, if the server is still running after *timeout* seconds, it will
-    send SIGTERM. After another timeout, it escalates to SIGKILL.
-
-    Returns True if the server was stopped by any means, False if stopping it
-    failed (on Windows).
-    """
+def kernel_request(server_info, path='/login', method='GET', body=None, headers=None, timeout=5, log=None):
+    """Query a notebook server in a separate process."""
     from tornado import gen
-    from tornado.httpclient import AsyncHTTPClient, HTTPClient, HTTPRequest
+    from tornado.httpclient import AsyncHTTPClient, HTTPClient, HTTPRequest, HTTPError
     from tornado.netutil import Resolver
     url = server_info['url']
-    pid = server_info['pid']
     resolver = None
 
     # UNIX Socket handling.
@@ -468,12 +459,44 @@ def resolve(self, host, port, *args, **kwargs):
 
         resolver = UnixSocketResolver(resolver=Resolver())
 
-    req = HTTPRequest(url + 'api/shutdown', method='POST', body=b'', headers={
-        'Authorization': 'token ' + server_info['token']
-    })
-    if log: log.debug("POST request to %sapi/shutdown", url)
-    AsyncHTTPClient.configure(None, resolver=resolver)
-    HTTPClient(AsyncHTTPClient).fetch(req)
+    fullurl = urljoin(url, path)
+    headers = dict(headers) if headers is not None else {}
+    headers.setdefault('Authorization', 'token ' + server_info['token'])
+    req = HTTPRequest(fullurl,
+        method=method, body=body, headers=headers,
+        follow_redirects=True,
+        decompress_response=True,
+        allow_nonstandard_methods=False,
+        validate_cert=False
+    )
+    if log: log.debug("%s request to %s", method, fullurl)
+
+    savedAsyncHTTPClient = AsyncHTTPClient._save_configuration()
+    try:
+        AsyncHTTPClient.configure(None, resolver=resolver)
+        response = HTTPClient(AsyncHTTPClient).fetch(req)
+    except HTTPError as e:
+        if e.response is not None:
+            response = e.response
+        else:
+            raise
+    finally:
+        AsyncHTTPClient._restore_configuration(savedAsyncHTTPClient)
+
+    return response
+
+
+def shutdown_server(server_info, timeout=5, log=None):
+    """Shutdown a notebook server in a separate process.
+    *server_info* should be a dictionary as produced by list_running_servers().
+    Will first try to request shutdown using /api/shutdown .
+    On Unix, if the server is still running after *timeout* seconds, it will
+    send SIGTERM. After another timeout, it escalates to SIGKILL.
+    Returns True if the server was stopped by any means, False if stopping it
+    failed (on Windows).
+    """
+    pid = server_info['pid']
+    kernel_request(server_info, path='api/shutdown', method='POST', body=b'', timeout=timeout, log=log)
 
     # Poll to see if it shut down.
     for _ in range(timeout*10):
@@ -1399,18 +1422,25 @@ def _update_mathjax_config(self, change):
               "sent by the upstream reverse proxy. Necessary if the proxy handles SSL"))
     )
 
+    file_id = Unicode()
+
+    @default('file_id')
+    def _default_file_id(self):
+        return str(hash(self.sock) if self.sock else self.port)
+        return str(int_hash)
+
     info_file = Unicode()
 
     @default('info_file')
     def _default_info_file(self):
-        info_file = "nbserver-%s.json" % os.getpid()
+        info_file = "nbserver-%s.json" % self.file_id
         return os.path.join(self.runtime_dir, info_file)
 
     browser_open_file = Unicode()
 
     @default('browser_open_file')
     def _default_browser_open_file(self):
-        basename = "nbserver-%s-open.html" % os.getpid()
+        basename = "nbserver-%s-open.html" % self.file_id
         return os.path.join(self.runtime_dir, basename)
 
     pylab = Unicode('disabled', config=True,
@@ -2215,7 +2245,8 @@ def start(self):
                  "resources section at https://jupyter.org/community.html."))
 
         self.write_server_info_file()
-        self.write_browser_open_file()
+        if not self.sock:
+            self.write_browser_open_file()
 
         if (self.open_browser or self.file_to_run) and not self.sock:
             self.launch_browser()
@@ -2294,16 +2325,36 @@ def list_running_servers(runtime_dir=None):
             with io.open(os.path.join(runtime_dir, file_name), encoding='utf-8') as f:
                 info = json.load(f)
 
-            # Simple check whether that process is really still running
+            # Actively check whether that process is really available via an HTTP request
             # Also remove leftover files from IPython 2.x without a pid field
-            if ('pid' in info) and check_pid(info['pid']):
-                yield info
-            else:
-                # If the process has died, try to delete its info file
+            response = kernel_request(info, path=url_path_join(info.get('base_url') or '/','/api'))
+            def flush_info_file():
                 try:
                     os.unlink(os.path.join(runtime_dir, file_name))
                 except OSError:
-                    pass  # TODO: This should warn or log or something
+                    pass
+
+            version_dict = {}
+            try:
+                version_dict.update(json.loads(response.body.decode()))
+            except:
+                flush_info_file()
+            else:
+                received_version = version_dict.get('version', '0.0.0')
+                msg_special = "WARNING: Jupyter Notebook HIGHER VERSION was detected: %s, current:%s" %(
+                    received_version, __version__)
+                msg_upgrade = "Jupyter Notebook Upgrade REQUIRED: >=%s, got:%s(info file removed)" % (
+                    __version__, received_version)
+
+                if JUPYTER_NOTEBOOK_TAG != version_dict.get('module'):
+                    raise MultipleInstanceError(msg_upgrade)
+                else:
+                    if __version__ < received_version:
+                        print(msg_special)
+                    yield info
+
+
+
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------

notebook/tests/selenium/conftest.py

@@ -11,6 +11,7 @@
 import time
 from urllib.parse import urljoin
 
+from notebook.utils import available_port
 from selenium.webdriver import Firefox, Remote, Chrome
 from .utils import Notebook
 
@@ -41,6 +42,7 @@ def notebook_server():
         nbdir = info['nbdir'] = pjoin(td, 'notebooks')
         os.makedirs(pjoin(nbdir, u'sub ∂ir1', u'sub ∂ir 1a'))
         os.makedirs(pjoin(nbdir, u'sub ∂ir2', u'sub ∂ir 1b'))
+        port = available_port()
 
         info['extra_env'] = {
             'JUPYTER_CONFIG_DIR': pjoin(td, 'jupyter_config'),
@@ -52,6 +54,7 @@ def notebook_server():
 
         command = [sys.executable, '-m', 'notebook',
                    '--no-browser',
+                   '--port=%s' % port,
                    '--notebook-dir', nbdir,
                    # run with a base URL that would be escaped,
                    # to test that we don't double-escape URLs
@@ -60,7 +63,7 @@ def notebook_server():
         print("command=", command)
         proc = info['popen'] = Popen(command, cwd=nbdir, env=env)
         info_file_path = pjoin(td, 'jupyter_runtime',
-                               'nbserver-%i.json' % proc.pid)
+                               'nbserver-%s.json' % port)
         info.update(_wait_for_server(proc, info_file_path))
 
         print("Notebook server info:", info)

notebook/tests/test_notebookapp.py

@@ -30,21 +30,40 @@ def test_help_output():
     check_help_all_output('notebook')
 
 def test_server_info_file():
+    import threading, tornado.ioloop as iom, tornado.platform.asyncio as torio
     td = TemporaryDirectory()
+
+    if iom.asyncio is not None:
+        iom.asyncio.set_event_loop_policy(torio.AnyThreadEventLoopPolicy())
+        iom.IOLoop.configure("tornado.platform.asyncio.AsyncIOLoop")
+
     nbapp = NotebookApp(runtime_dir=td.name, log=logging.getLogger())
-    def get_servers():
-        return list(notebookapp.list_running_servers(nbapp.runtime_dir))
     nbapp.initialize(argv=[])
+    nbapp.io_loop = iom.IOLoop.current()
+    nbapp.open_browser = False
+    super(NotebookApp, nbapp).start()
     nbapp.write_server_info_file()
-    servers = get_servers()
-    nt.assert_equal(len(servers), 1)
-    nt.assert_equal(servers[0]['port'], nbapp.port)
-    nt.assert_equal(servers[0]['url'], nbapp.connection_url)
-    nbapp.remove_server_info_file()
-    nt.assert_equal(get_servers(), [])
 
-    # The ENOENT error should be silenced.
-    nbapp.remove_server_info_file()
+    def check_thread():
+        try:
+            servers = list(notebookapp.list_running_servers(nbapp.runtime_dir))
+            nt.assert_equal(len(servers), 1)
+            nt.assert_equal(servers[0]['port'], nbapp.port)
+            nt.assert_equal(servers[0]['url'], nbapp.connection_url)
+        finally:
+            nbapp.stop()
+
+    nbapp.io_loop.add_callback(nbapp.io_loop.run_in_executor, executor=None, func=check_thread)
+
+    if sys.platform.startswith("win"):
+        pc = iom.PeriodicCallback(lambda: None, 5000)
+        pc.start()
+    try:
+        nbapp.io_loop.start()
+    except KeyboardInterrupt:
+        print("Interrupted...")
+    finally:
+        nbapp.remove_server_info_file()
 
 def test_nb_dir():
     with TemporaryDirectory() as td:
@@ -200,4 +219,4 @@ def test_run(self):
         def test_list_running_sock_servers(self):
             servers = list(notebookapp.list_running_servers())
             assert len(servers) >= 1
-            assert self.sock in {info['sock'] for info in servers}
+            assert self.sock in {info['sock'] for info in servers}

notebook/utils.py

@@ -22,6 +22,7 @@
 # tornado.concurrent.Future is asyncio.Future
 # in tornado >=5 with Python 3
 from tornado.concurrent import Future as TornadoFuture
+from tornado.netutil import bind_sockets
 from tornado import gen
 from ipython_genutils import py3compat
 
@@ -30,6 +31,11 @@
 UF_HIDDEN = getattr(stat, 'UF_HIDDEN', 32768)
 
 
+def available_port(host='127.0.0.1'):
+    sockets = bind_sockets(0, host)
+    return sockets[0].getsockname()[:2][1]
+
+
 def exists(path):
     """Replacement for `os.path.exists` which works for host mapped volumes
     on Windows containers