improve documentation

Signed-off-by: Liu Jiang <[email protected]>

src/completion_queue.rs

@@ -87,14 +87,17 @@ impl<'ring> CompletionQueue<'ring> {
         CQEsBlocking::new(self.ring, wait_for)
     }
 
+    /// Returns how many descriptors are ready for processing on the completion queue.
     pub fn ready(&self) -> u32 {
         unsafe { uring_sys::io_uring_cq_ready(self.ring.as_ptr()) }
     }
 
+    /// Returns true if the eventfd notification is currently enabled.
     pub fn eventfd_enabled(&self) -> bool {
         unsafe { uring_sys::io_uring_cq_eventfd_enabled(self.ring.as_ptr()) }
     }
 
+    /// Toggle eventfd notification on or off, if an eventfd is registered with the ring.
     pub fn eventfd_toggle(&mut self, enabled: bool) -> io::Result<()> {
         resultify(unsafe { uring_sys::io_uring_cq_eventfd_toggle(self.ring.as_ptr(), enabled) })?;
         Ok(())

src/lib.rs

@@ -103,9 +103,16 @@ bitflags::bitflags! {
         /// Force the kernel thread created with `SQPOLL` to be bound to the CPU used by the
         /// `SubmissionQueue`. Requires `SQPOLL` set.
         const SQ_AFF    = 1 << 2;   /* sq_thread_cpu is valid */
-
+        /// Create the completion queue with struct io_uring_params.cq_entries entries.
+        /// The value must be greater than entries, and may be rounded up to the next power-of-two.
         const CQSIZE    = 1 << 3;
+        /// Clamp the values for SQ or CQ ring size to the max values instead of returning -EINVAL.
         const CLAMP     = 1 << 4;
+        /// Share the asynchronous backend (kernel work thread) with an existing io_uring instance.
+        ///
+        /// If ATTACH_WQ is set, io_uring_params::wq_fd should be a valid io_uring fd, io-wq of
+        /// which will be shared with the newly created io_uring instance. If the flag is set
+        /// but it can't share io-wq, it fails.
         const ATTACH_WQ = 1 << 5;
     }
 }
@@ -232,6 +239,7 @@ impl IoUring {
         )
     }
 
+    /// Returns a probe structure to detect supported IO operations.
     pub fn probe(&mut self) -> io::Result<Probe> {
         Probe::for_ring(&mut self.ring)
     }
@@ -255,12 +263,33 @@ impl IoUring {
 
     /// Submit all prepared [`SQE`]s to the kernel and wait until at least `wait_for` events have
     /// completed.
+    ///
+    /// # Return value
+    /// - the number of submitted events, it's safe to reuse SQE entries in the ring. This is true
+    ///   even if the actual IO submission had to be punted to async context, which means that the
+    ///   SQE may in fact not have been submitted yet.
+    /// - an [`io::Error`](std::io::Result) variant if this function encounters any IO errors.
     pub fn submit_sqes_and_wait(&mut self, wait_for: u32) -> io::Result<u32> {
         self.sq().submit_and_wait(wait_for)
     }
 
     /// Submit all prepared [`SQE`]s to the kernel and wait until at least `wait_for` events have
     /// completed or `duration` has passed.
+    ///
+    /// # Return value
+    /// - the number of submitted events, it's safe to reuse SQE entries in the ring. This is true
+    ///   even if the actual IO submission had to be punted to async context, which means that the
+    ///   SQE may in fact not have been submitted yet.
+    /// - an [`io::Error`](std::io::Result) variant if this function encounters any IO errors.
+    ///
+    /// # Note
+    /// Due to the way timeout is implemented, there are two possible flaws:
+    /// - the timeout is unreliable. When all submission queue is full, it fallbacks to submit()
+    ///   silently.
+    /// - the returned value may be bigger than expectation. There may be one extra descriptor
+    ///   consumed by the timeout mechanism. The user data of descriptor consumed by timeout is
+    ///   set to [`LIBURING_UDATA_TIMEOUT`](uring_sys::LIBURING_UDATA_TIMEOUT)(u64::MAX), so this
+    ///   special value is reserved.
     pub fn submit_sqes_and_wait_with_timeout(
         &mut self,
         wait_for: u32,
@@ -292,6 +321,10 @@ impl IoUring {
     }
 
     /// Block until a [`CQE`] is ready or timeout.
+    ///
+    /// # Safety
+    /// The timeout is implemented by adding an IORING_OP_TIMEOUT event to the submission queue,
+    /// so it touches both the submission and completion queue and not multi-thread safe.
     pub fn wait_for_cqe_with_timeout(&mut self, duration: Duration) -> io::Result<CQE> {
         let ts = uring_sys::__kernel_timespec {
             tv_sec: duration.as_secs() as _,
@@ -351,26 +384,32 @@ impl IoUring {
         &mut self.ring
     }
 
+    /// Returns how many descriptors are ready for processing on the completion queue.
     pub fn cq_ready(&mut self) -> u32 {
         self.cq().ready()
     }
 
+    /// Returns the numbers of ready event descriptors on the submission queue.
     pub fn sq_ready(&mut self) -> u32 {
         self.sq().ready()
     }
 
+    /// Returns the numbers of available event descriptors on the submission queue.
     pub fn sq_space_left(&mut self) -> u32 {
         self.sq().space_left()
     }
 
+    /// Returns true if the eventfd notification is currently enabled.
     pub fn cq_eventfd_enabled(&mut self) -> bool {
         self.cq().eventfd_enabled()
     }
 
+    /// Toggle eventfd notification on or off, if an eventfd is registered with the ring.
     pub fn cq_eventfd_toggle(&mut self, enabled: bool) -> io::Result<()> {
         self.cq().eventfd_toggle(enabled)
     }
 
+    /// Returns the RawFd for the io_uring handle.
     pub fn raw_fd(&self) -> RawFd {
         self.ring.ring_fd
     }

src/probe.rs

@@ -26,6 +26,7 @@ impl Probe {
         }
     }
 
+    /// Check whether an operation is supported by this kernel version's io-uring interface.
     pub fn supports(&self, op: uring_sys::IoRingOp) -> bool {
         unsafe { uring_sys::io_uring_opcode_supported(self.probe.as_ptr(), op as _) != 0 }
     }

src/registrar/mod.rs

@@ -262,6 +262,7 @@ unsafe impl<'ring> Send for Registrar<'ring> {}
 unsafe impl<'ring> Sync for Registrar<'ring> {}
 
 #[derive(Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Clone, Copy)]
+/// An identity for a registered credential.
 pub struct Personality {
     pub(crate) id: u16,
 }

src/sqe.rs

@@ -618,14 +618,19 @@ bitflags::bitflags! {
         const FIXED_FILE    = 1 << 0;   /* use fixed fileset */
         /// Submit this event only after completing all ongoing submission events.
         const IO_DRAIN      = 1 << 1;   /* issue after inflight IO */
-        /// Force the next submission event to wait until this event has completed sucessfully.
+        /// Force the next submission event to wait until this event has completed successfully.
         ///
         /// An event's link only applies to the next event, but link chains can be
         /// arbitrarily long.
         const IO_LINK       = 1 << 2;   /* next IO depends on this one */
-
+        /// Force the next submission event to wait until this event has completed.
+        ///
+        /// An event's link only applies to the next event, but link chains can be arbitrarily long.
+        /// The next submission event will be executed no matter current event succeeds or fails.
         const IO_HARDLINK   = 1 << 3;
+        /// Execute the event in asynchronous mode without trying non-blocking mode first.
         const ASYNC         = 1 << 4;
+
         const BUFFER_SELECT = 1 << 5;
     }
 }
@@ -738,6 +743,9 @@ impl<'ring> Iterator for SQEs<'ring> {
 }
 
 /// An Iterator of [`SQE`]s which will be hard linked together.
+///
+/// All HardLinked objects must be dropped before submitting the submission queue to ensure
+/// correctly handling of the IO chain.
 pub struct HardLinked<'ring, 'a> {
     sqes: &'a mut SQEs<'ring>,
 }
@@ -770,6 +778,10 @@ impl<'ring> Drop for HardLinked<'ring, '_> {
     }
 }
 
+/// Represent a non-tail event descriptor on an hardly linked IO chain.
+///
+/// All HardLinkedSQE objects must be dropped before submitting the submission queue to ensure
+/// correctly handling of the IO_HARDLINK flag.
 pub struct HardLinkedSQE<'ring> {
     sqe: SQE<'ring>,
     is_final: bool,
@@ -789,6 +801,10 @@ impl<'ring> DerefMut for HardLinkedSQE<'ring> {
     }
 }
 
+// TODO: any better way to set the IO_HARDLINK flag?
+// If submit() is called before dropping the HardLinkedSQE object, it may caused race windows
+// under which the kernel observes malformed IO chains. This type of race window will be very hard
+// to root cause.
 impl<'ring> Drop for HardLinkedSQE<'ring> {
     fn drop(&mut self) {
         if !self.is_final {
@@ -798,6 +814,9 @@ impl<'ring> Drop for HardLinkedSQE<'ring> {
 }
 
 /// An Iterator of [`SQE`]s which will be soft linked together.
+///
+/// All SoftLinked objects must be dropped before submitting the submission queue to ensure
+/// correctly handling of the IO chain.
 pub struct SoftLinked<'ring, 'a> {
     sqes: &'a mut SQEs<'ring>,
 }
@@ -830,6 +849,10 @@ impl<'ring> Drop for SoftLinked<'ring, '_> {
     }
 }
 
+/// Represent a non-tail event descriptor on an softly linked IO chain.
+///
+/// All SoftLinkedSQE objects must be dropped before submitting the submission queue to ensure
+/// correctly handling of the IO_HARDLINK flag.
 pub struct SoftLinkedSQE<'ring> {
     sqe: SQE<'ring>,
     is_final: bool,
@@ -849,6 +872,10 @@ impl<'ring> DerefMut for SoftLinkedSQE<'ring> {
     }
 }
 
+// TODO: any better way to set the IO_LINK flag?
+// If submit() is called before dropping the SoftLinkedSQE object, it may caused race windows
+// under which the kernel observes malformed IO chains. This type of race window will be very hard
+// to root cause.
 impl<'ring> Drop for SoftLinkedSQE<'ring> {
     fn drop(&mut self) {
         if !self.is_final {

src/submission_queue.rs

@@ -90,15 +90,43 @@ impl<'ring> SubmissionQueue<'ring> {
 
     /// Submit all events in the queue. Returns the number of submitted events.
     ///
-    /// If this function encounters any IO errors an [`io::Error`](std::io::Result) variant is returned.
+    /// # Return value
+    /// - the number of submitted events, it's safe to reuse SQE entries in the ring. This is true
+    ///   even if the actual IO submission had to be punted to async context, which means that the
+    ///   SQE may in fact not have been submitted yet.
+    /// - an [`io::Error`](std::io::Result) variant if this function encounters any IO errors.
     pub fn submit(&mut self) -> io::Result<u32> {
         resultify(unsafe { uring_sys::io_uring_submit(self.ring.as_ptr()) })
     }
 
+    /// Submit all events in the queue and wait for `wait_for` event completions before returning.
+    ///
+    /// # Return value
+    /// - the number of submitted events, it's safe to reuse SQE entries in the ring. This is true
+    ///   even if the actual IO submission had to be punted to async context, which means that the
+    ///   SQE may in fact not have been submitted yet.
+    /// - an [`io::Error`](std::io::Result) variant if this function encounters any IO errors.
     pub fn submit_and_wait(&mut self, wait_for: u32) -> io::Result<u32> {
         resultify(unsafe { uring_sys::io_uring_submit_and_wait(self.ring.as_ptr(), wait_for as _) })
     }
 
+    /// Submit all events in the queue and wait for `wait_for` event completions before returning,
+    /// timeout after waiting for `duration`.
+    ///
+    /// # Return value
+    /// - the number of submitted events, it's safe to reuse SQE entries in the ring. This is true
+    ///   even if the actual IO submission had to be punted to async context, which means that the
+    ///   SQE may in fact not have been submitted yet.
+    /// - an [`io::Error`](std::io::Result) variant if this function encounters any IO errors.
+    ///
+    /// # Note
+    /// Due to the way timeout is implemented, there are two possible flaws:
+    /// - the timeout is unreliable. When all submission queue is full, it fallbacks to submit()
+    ///   silently.
+    /// - the returned value may be bigger than expectation. There may be one extra descriptor
+    ///   consumed by the timeout mechanism. The user data of descriptor consumed by timeout is
+    ///   set to [`LIBURING_UDATA_TIMEOUT`](uring_sys::LIBURING_UDATA_TIMEOUT)(u64::MAX), so this
+    ///   special value is reserved.
     pub fn submit_and_wait_with_timeout(
         &mut self,
         wait_for: u32,
@@ -111,7 +139,6 @@ impl<'ring> SubmissionQueue<'ring> {
 
         loop {
             if let Some(mut sqe) = self.prepare_sqe() {
-                sqe.clear();
                 unsafe {
                     sqe.prep_timeout(&ts, 0, crate::sqe::TimeoutFlags::empty());
                     sqe.set_user_data(uring_sys::LIBURING_UDATA_TIMEOUT);
@@ -126,10 +153,12 @@ impl<'ring> SubmissionQueue<'ring> {
         }
     }
 
+    /// Returns the numbers of ready event descriptors on the submission queue.
     pub fn ready(&self) -> u32 {
         unsafe { uring_sys::io_uring_sq_ready(self.ring.as_ptr()) as u32 }
     }
 
+    /// Returns the numbers of available event descriptors on the submission queue.
     pub fn space_left(&self) -> u32 {
         unsafe { uring_sys::io_uring_sq_space_left(self.ring.as_ptr()) as u32 }
     }