diff --git a/lib/server.js b/lib/server.js
index 783ed105..e5ab5256 100644
--- a/lib/server.js
+++ b/lib/server.js
@@ -38,10 +38,14 @@ module.exports = class Server extends EventEmitter {
     this._announcer = null
     this._connects = new Map()
     this._holepunches = []
-    this._listening = false
+    this._listening = null
     this._closing = null
   }
 
+  get listening () {
+    return this._listening !== null
+  }
+
   get publicKey () {
     return this._keyPair && this._keyPair.publicKey
   }
@@ -54,12 +58,14 @@ module.exports = class Server extends EventEmitter {
     this.emit('connection', encryptedSocket)
   }
 
-  suspend () {
+  async suspend () {
+    if (this._listening !== null) await this._listening
     this.suspended = true
     return this._announcer ? this._announcer.suspend() : Promise.resolve()
   }
 
-  resume () {
+  async resume () {
+    if (this._listening !== null) await this._listening
     this.suspended = false
     return this._announcer ? this._announcer.resume() : Promise.resolve()
   }
@@ -80,13 +86,35 @@ module.exports = class Server extends EventEmitter {
     return this._closing
   }
 
+  _gc () {
+    this.dht.listening.delete(this)
+    if (this.target) this.dht._router.delete(this.target)
+  }
+
+  async _stopListening () {
+    try {
+      if (this._announcer) await this._announcer.stop()
+    } catch {
+      // ignore
+    }
+
+    this._announcer = null
+    this._listening = null
+    this._keyPair = null
+  }
+
   async _close () {
-    this.closed = true
+    if (this._listening === null) {
+      this.closed = true
+      this.emit('close')
+      return
+    }
 
-    if (!this._listening) return
+    try {
+      await this._listening
+    } catch {}
 
-    this.dht.listening.delete(this)
-    this.dht._router.delete(this.target)
+    this._gc()
 
     while (this._holepunches.length > 0) {
       const h = this._holepunches.pop()
@@ -98,16 +126,22 @@ module.exports = class Server extends EventEmitter {
 
     this._connects.clear()
 
-    await this._announcer.stop()
-    this._announcer = null
+    await this._stopListening()
 
+    this.closed = true
     this.emit('close')
   }
 
   async listen (keyPair = this.dht.defaultKeyPair, opts = {}) {
-    if (this._listening) throw ALREADY_LISTENING()
+    if (this._listening !== null) throw ALREADY_LISTENING()
     if (this.dht.destroyed) throw NODE_DESTROYED()
 
+    this._listening = this._listen(keyPair, opts)
+    await this._listening
+    return this
+  }
+
+  async _listen (keyPair, opts) {
     // From now on, the DHT object which created me is responsible for closing me
     this.dht.listening.add(this)
 
@@ -126,17 +160,14 @@ module.exports = class Server extends EventEmitter {
       onpeerholepunch: this._onpeerholepunch.bind(this)
     })
 
-    this._listening = true
-
     // warm it up for now
     this._localAddresses().catch(safetyCatch)
 
     try {
       await this._announcer.start()
     } catch (err) {
-      await this._announcer.stop()
-      this._announcer = null
-      this._listening = false
+      await this._stopListening()
+      this._gc()
       throw err
     }
 
@@ -146,11 +177,9 @@ module.exports = class Server extends EventEmitter {
     if (this._closing) return
     if (this.dht.destroyed) throw NODE_DESTROYED()
 
-    this.emit('listening')
-
     if (this.pool) this.pool._attachServer(this)
 
-    return this
+    this.emit('listening')
   }
 
   refresh () {
@@ -410,7 +439,7 @@ module.exports = class Server extends EventEmitter {
     const h = await p
     if (!h) return null
 
-    if (this.closed) return null
+    if (this._closing !== null) return null
 
     return { socket: h.puncher && h.puncher.socket, noise: h.reply }
   }
@@ -419,7 +448,7 @@ module.exports = class Server extends EventEmitter {
     const h = id < this._holepunches.length ? this._holepunches[id] : null
     if (!h) return null
 
-    if (!peerAddress || this.closed) return null
+    if (!peerAddress || this._closing !== null) return null
 
     const p = h.puncher
     if (!p || !p.socket) return this._abort(h) // not opened