bytes/
bytes.rs

Help
1use core::iter::FromIterator;
2use core::ops::{Deref, RangeBounds};
3use core::{cmp, fmt, hash, mem, ptr, slice, usize};
4
5use alloc::{
6    alloc::{dealloc, Layout},
7    borrow::Borrow,
8    boxed::Box,
9    string::String,
10    vec::Vec,
11};
12
13use crate::buf::IntoIter;
14#[allow(unused)]
15use crate::loom::sync::atomic::AtomicMut;
16use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
17use crate::Buf;
18
19/// A cheaply cloneable and sliceable chunk of contiguous memory.
20///
21/// `Bytes` is an efficient container for storing and operating on contiguous
22/// slices of memory. It is intended for use primarily in networking code, but
23/// could have applications elsewhere as well.
24///
25/// `Bytes` values facilitate zero-copy network programming by allowing multiple
26/// `Bytes` objects to point to the same underlying memory.
27///
28/// `Bytes` does not have a single implementation. It is an interface, whose
29/// exact behavior is implemented through dynamic dispatch in several underlying
30/// implementations of `Bytes`.
31///
32/// All `Bytes` implementations must fulfill the following requirements:
33/// - They are cheaply cloneable and thereby shareable between an unlimited amount
34///   of components, for example by modifying a reference count.
35/// - Instances can be sliced to refer to a subset of the the original buffer.
36///
37/// ```
38/// use bytes::Bytes;
39///
40/// let mut mem = Bytes::from("Hello world");
41/// let a = mem.slice(0..5);
42///
43/// assert_eq!(a, "Hello");
44///
45/// let b = mem.split_to(6);
46///
47/// assert_eq!(mem, "world");
48/// assert_eq!(b, "Hello ");
49/// ```
50///
51/// # Memory layout
52///
53/// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used
54/// to track information about which segment of the underlying memory the
55/// `Bytes` handle has access to.
56///
57/// `Bytes` keeps both a pointer to the shared state containing the full memory
58/// slice and a pointer to the start of the region visible by the handle.
59/// `Bytes` also tracks the length of its view into the memory.
60///
61/// # Sharing
62///
63/// `Bytes` contains a vtable, which allows implementations of `Bytes` to define
64/// how sharing/cloning is implemented in detail.
65/// When `Bytes::clone()` is called, `Bytes` will call the vtable function for
66/// cloning the backing storage in order to share it behind between multiple
67/// `Bytes` instances.
68///
69/// For `Bytes` implementations which refer to constant memory (e.g. created
70/// via `Bytes::from_static()`) the cloning implementation will be a no-op.
71///
72/// For `Bytes` implementations which point to a reference counted shared storage
73/// (e.g. an `Arc<[u8]>`), sharing will be implemented by increasing the
74/// the reference count.
75///
76/// Due to this mechanism, multiple `Bytes` instances may point to the same
77/// shared memory region.
78/// Each `Bytes` instance can point to different sections within that
79/// memory region, and `Bytes` instances may or may not have overlapping views
80/// into the memory.
81///
82/// The following diagram visualizes a scenario where 2 `Bytes` instances make
83/// use of an `Arc`-based backing storage, and provide access to different views:
84///
85/// ```text
86///
87///    Arc ptrs                   ┌─────────┐
88///    ________________________ / │ Bytes 2 │
89///   /                           └─────────┘
90///  /          ┌───────────┐     |         |
91/// |_________/ │  Bytes 1  │     |         |
92/// |           └───────────┘     |         |
93/// |           |           | ___/ data     | tail
94/// |      data |      tail |/              |
95/// v           v           v               v
96/// ┌─────┬─────┬───────────┬───────────────┬─────┐
97/// │ Arc │     │           │               │     │
98/// └─────┴─────┴───────────┴───────────────┴─────┘
99/// ```
100pub struct Bytes {
101    ptr: *const u8,
102    len: usize,
103    // inlined "trait object"
104    data: AtomicPtr<()>,
105    vtable: &'static Vtable,
106}
107
108pub(crate) struct Vtable {
109    /// fn(data, ptr, len)
110    pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
111    /// fn(data, ptr, len)
112    ///
113    /// takes `Bytes` to value
114    pub to_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec<u8>,
115    /// fn(data, ptr, len)
116    pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize),
117}
118
119impl Bytes {
120    /// Creates a new empty `Bytes`.
121    ///
122    /// This will not allocate and the returned `Bytes` handle will be empty.
123    ///
124    /// # Examples
125    ///
126    /// ```
127    /// use bytes::Bytes;
128    ///
129    /// let b = Bytes::new();
130    /// assert_eq!(&b[..], b"");
131    /// ```
132    #[inline]
133    #[cfg(not(all(loom, test)))]
134    pub const fn new() -> Bytes {
135        // Make it a named const to work around
136        // "unsizing casts are not allowed in const fn"
137        const EMPTY: &[u8] = &[];
138        Bytes::from_static(EMPTY)
139    }
140
141    #[cfg(all(loom, test))]
142    pub fn new() -> Bytes {
143        const EMPTY: &[u8] = &[];
144        Bytes::from_static(EMPTY)
145    }
146
147    /// Creates a new `Bytes` from a static slice.
148    ///
149    /// The returned `Bytes` will point directly to the static slice. There is
150    /// no allocating or copying.
151    ///
152    /// # Examples
153    ///
154    /// ```
155    /// use bytes::Bytes;
156    ///
157    /// let b = Bytes::from_static(b"hello");
158    /// assert_eq!(&b[..], b"hello");
159    /// ```
160    #[inline]
161    #[cfg(not(all(loom, test)))]
162    pub const fn from_static(bytes: &'static [u8]) -> Bytes {
163        Bytes {
164            ptr: bytes.as_ptr(),
165            len: bytes.len(),
166            data: AtomicPtr::new(ptr::null_mut()),
167            vtable: &STATIC_VTABLE,
168        }
169    }
170
171    #[cfg(all(loom, test))]
172    pub fn from_static(bytes: &'static [u8]) -> Bytes {
173        Bytes {
174            ptr: bytes.as_ptr(),
175            len: bytes.len(),
176            data: AtomicPtr::new(ptr::null_mut()),
177            vtable: &STATIC_VTABLE,
178        }
179    }
180
181    /// Returns the number of bytes contained in this `Bytes`.
182    ///
183    /// # Examples
184    ///
185    /// ```
186    /// use bytes::Bytes;
187    ///
188    /// let b = Bytes::from(&b"hello"[..]);
189    /// assert_eq!(b.len(), 5);
190    /// ```
191    #[inline]
192    pub const fn len(&self) -> usize {
193        self.len
194    }
195
196    /// Returns true if the `Bytes` has a length of 0.
197    ///
198    /// # Examples
199    ///
200    /// ```
201    /// use bytes::Bytes;
202    ///
203    /// let b = Bytes::new();
204    /// assert!(b.is_empty());
205    /// ```
206    #[inline]
207    pub const fn is_empty(&self) -> bool {
208        self.len == 0
209    }
210
211    /// Creates `Bytes` instance from slice, by copying it.
212    pub fn copy_from_slice(data: &[u8]) -> Self {
213        data.to_vec().into()
214    }
215
216    /// Returns a slice of self for the provided range.
217    ///
218    /// This will increment the reference count for the underlying memory and
219    /// return a new `Bytes` handle set to the slice.
220    ///
221    /// This operation is `O(1)`.
222    ///
223    /// # Examples
224    ///
225    /// ```
226    /// use bytes::Bytes;
227    ///
228    /// let a = Bytes::from(&b"hello world"[..]);
229    /// let b = a.slice(2..5);
230    ///
231    /// assert_eq!(&b[..], b"llo");
232    /// ```
233    ///
234    /// # Panics
235    ///
236    /// Requires that `begin <= end` and `end <= self.len()`, otherwise slicing
237    /// will panic.
238    pub fn slice(&self, range: impl RangeBounds<usize>) -> Bytes {
239        use core::ops::Bound;
240
241        let len = self.len();
242
243        let begin = match range.start_bound() {
244            Bound::Included(&n) => n,
245            Bound::Excluded(&n) => n + 1,
246            Bound::Unbounded => 0,
247        };
248
249        let end = match range.end_bound() {
250            Bound::Included(&n) => n.checked_add(1).expect("out of range"),
251            Bound::Excluded(&n) => n,
252            Bound::Unbounded => len,
253        };
254
255        assert!(
256            begin <= end,
257            "range start must not be greater than end: {:?} <= {:?}",
258            begin,
259            end,
260        );
261        assert!(
262            end <= len,
263            "range end out of bounds: {:?} <= {:?}",
264            end,
265            len,
266        );
267
268        if end == begin {
269            return Bytes::new();
270        }
271
272        let mut ret = self.clone();
273
274        ret.len = end - begin;
275        ret.ptr = unsafe { ret.ptr.add(begin) };
276
277        ret
278    }
279
280    /// Returns a slice of self that is equivalent to the given `subset`.
281    ///
282    /// When processing a `Bytes` buffer with other tools, one often gets a
283    /// `&[u8]` which is in fact a slice of the `Bytes`, i.e. a subset of it.
284    /// This function turns that `&[u8]` into another `Bytes`, as if one had
285    /// called `self.slice()` with the offsets that correspond to `subset`.
286    ///
287    /// This operation is `O(1)`.
288    ///
289    /// # Examples
290    ///
291    /// ```
292    /// use bytes::Bytes;
293    ///
294    /// let bytes = Bytes::from(&b"012345678"[..]);
295    /// let as_slice = bytes.as_ref();
296    /// let subset = &as_slice[2..6];
297    /// let subslice = bytes.slice_ref(&subset);
298    /// assert_eq!(&subslice[..], b"2345");
299    /// ```
300    ///
301    /// # Panics
302    ///
303    /// Requires that the given `sub` slice is in fact contained within the
304    /// `Bytes` buffer; otherwise this function will panic.
305    pub fn slice_ref(&self, subset: &[u8]) -> Bytes {
306        // Empty slice and empty Bytes may have their pointers reset
307        // so explicitly allow empty slice to be a subslice of any slice.
308        if subset.is_empty() {
309            return Bytes::new();
310        }
311
312        let bytes_p = self.as_ptr() as usize;
313        let bytes_len = self.len();
314
315        let sub_p = subset.as_ptr() as usize;
316        let sub_len = subset.len();
317
318        assert!(
319            sub_p >= bytes_p,
320            "subset pointer ({:p}) is smaller than self pointer ({:p})",
321            subset.as_ptr(),
322            self.as_ptr(),
323        );
324        assert!(
325            sub_p + sub_len <= bytes_p + bytes_len,
326            "subset is out of bounds: self = ({:p}, {}), subset = ({:p}, {})",
327            self.as_ptr(),
328            bytes_len,
329            subset.as_ptr(),
330            sub_len,
331        );
332
333        let sub_offset = sub_p - bytes_p;
334
335        self.slice(sub_offset..(sub_offset + sub_len))
336    }
337
338    /// Splits the bytes into two at the given index.
339    ///
340    /// Afterwards `self` contains elements `[0, at)`, and the returned `Bytes`
341    /// contains elements `[at, len)`.
342    ///
343    /// This is an `O(1)` operation that just increases the reference count and
344    /// sets a few indices.
345    ///
346    /// # Examples
347    ///
348    /// ```
349    /// use bytes::Bytes;
350    ///
351    /// let mut a = Bytes::from(&b"hello world"[..]);
352    /// let b = a.split_off(5);
353    ///
354    /// assert_eq!(&a[..], b"hello");
355    /// assert_eq!(&b[..], b" world");
356    /// ```
357    ///
358    /// # Panics
359    ///
360    /// Panics if `at > len`.
361    #[must_use = "consider Bytes::truncate if you don't need the other half"]
362    pub fn split_off(&mut self, at: usize) -> Bytes {
363        assert!(
364            at <= self.len(),
365            "split_off out of bounds: {:?} <= {:?}",
366            at,
367            self.len(),
368        );
369
370        if at == self.len() {
371            return Bytes::new();
372        }
373
374        if at == 0 {
375            return mem::replace(self, Bytes::new());
376        }
377
378        let mut ret = self.clone();
379
380        self.len = at;
381
382        unsafe { ret.inc_start(at) };
383
384        ret
385    }
386
387    /// Splits the bytes into two at the given index.
388    ///
389    /// Afterwards `self` contains elements `[at, len)`, and the returned
390    /// `Bytes` contains elements `[0, at)`.
391    ///
392    /// This is an `O(1)` operation that just increases the reference count and
393    /// sets a few indices.
394    ///
395    /// # Examples
396    ///
397    /// ```
398    /// use bytes::Bytes;
399    ///
400    /// let mut a = Bytes::from(&b"hello world"[..]);
401    /// let b = a.split_to(5);
402    ///
403    /// assert_eq!(&a[..], b" world");
404    /// assert_eq!(&b[..], b"hello");
405    /// ```
406    ///
407    /// # Panics
408    ///
409    /// Panics if `at > len`.
410    #[must_use = "consider Bytes::advance if you don't need the other half"]
411    pub fn split_to(&mut self, at: usize) -> Bytes {
412        assert!(
413            at <= self.len(),
414            "split_to out of bounds: {:?} <= {:?}",
415            at,
416            self.len(),
417        );
418
419        if at == self.len() {
420            return mem::replace(self, Bytes::new());
421        }
422
423        if at == 0 {
424            return Bytes::new();
425        }
426
427        let mut ret = self.clone();
428
429        unsafe { self.inc_start(at) };
430
431        ret.len = at;
432        ret
433    }
434
435    /// Shortens the buffer, keeping the first `len` bytes and dropping the
436    /// rest.
437    ///
438    /// If `len` is greater than the buffer's current length, this has no
439    /// effect.
440    ///
441    /// The [`split_off`] method can emulate `truncate`, but this causes the
442    /// excess bytes to be returned instead of dropped.
443    ///
444    /// # Examples
445    ///
446    /// ```
447    /// use bytes::Bytes;
448    ///
449    /// let mut buf = Bytes::from(&b"hello world"[..]);
450    /// buf.truncate(5);
451    /// assert_eq!(buf, b"hello"[..]);
452    /// ```
453    ///
454    /// [`split_off`]: #method.split_off
455    #[inline]
456    pub fn truncate(&mut self, len: usize) {
457        if len < self.len {
458            // The Vec "promotable" vtables do not store the capacity,
459            // so we cannot truncate while using this repr. We *have* to
460            // promote using `split_off` so the capacity can be stored.
461            if self.vtable as *const Vtable == &PROMOTABLE_EVEN_VTABLE
462                || self.vtable as *const Vtable == &PROMOTABLE_ODD_VTABLE
463            {
464                drop(self.split_off(len));
465            } else {
466                self.len = len;
467            }
468        }
469    }
470
471    /// Clears the buffer, removing all data.
472    ///
473    /// # Examples
474    ///
475    /// ```
476    /// use bytes::Bytes;
477    ///
478    /// let mut buf = Bytes::from(&b"hello world"[..]);
479    /// buf.clear();
480    /// assert!(buf.is_empty());
481    /// ```
482    #[inline]
483    pub fn clear(&mut self) {
484        self.truncate(0);
485    }
486
487    #[inline]
488    pub(crate) unsafe fn with_vtable(
489        ptr: *const u8,
490        len: usize,
491        data: AtomicPtr<()>,
492        vtable: &'static Vtable,
493    ) -> Bytes {
494        Bytes {
495            ptr,
496            len,
497            data,
498            vtable,
499        }
500    }
501
502    // private
503
504    #[inline]
505    fn as_slice(&self) -> &[u8] {
506        unsafe { slice::from_raw_parts(self.ptr, self.len) }
507    }
508
509    #[inline]
510    unsafe fn inc_start(&mut self, by: usize) {
511        // should already be asserted, but debug assert for tests
512        debug_assert!(self.len >= by, "internal: inc_start out of bounds");
513        self.len -= by;
514        self.ptr = self.ptr.add(by);
515    }
516}
517
518// Vtable must enforce this behavior
519unsafe impl Send for Bytes {}
520unsafe impl Sync for Bytes {}
521
522impl Drop for Bytes {
523    #[inline]
524    fn drop(&mut self) {
525        unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) }
526    }
527}
528
529impl Clone for Bytes {
530    #[inline]
531    fn clone(&self) -> Bytes {
532        unsafe { (self.vtable.clone)(&self.data, self.ptr, self.len) }
533    }
534}
535
536impl Buf for Bytes {
537    #[inline]
538    fn remaining(&self) -> usize {
539        self.len()
540    }
541
542    #[inline]
543    fn chunk(&self) -> &[u8] {
544        self.as_slice()
545    }
546
547    #[inline]
548    fn advance(&mut self, cnt: usize) {
549        assert!(
550            cnt <= self.len(),
551            "cannot advance past `remaining`: {:?} <= {:?}",
552            cnt,
553            self.len(),
554        );
555
556        unsafe {
557            self.inc_start(cnt);
558        }
559    }
560
561    fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes {
562        if len == self.remaining() {
563            core::mem::replace(self, Bytes::new())
564        } else {
565            let ret = self.slice(..len);
566            self.advance(len);
567            ret
568        }
569    }
570}
571
572impl Deref for Bytes {
573    type Target = [u8];
574
575    #[inline]
576    fn deref(&self) -> &[u8] {
577        self.as_slice()
578    }
579}
580
581impl AsRef<[u8]> for Bytes {
582    #[inline]
583    fn as_ref(&self) -> &[u8] {
584        self.as_slice()
585    }
586}
587
588impl hash::Hash for Bytes {
589    fn hash<H>(&self, state: &mut H)
590    where
591        H: hash::Hasher,
592    {
593        self.as_slice().hash(state);
594    }
595}
596
597impl Borrow<[u8]> for Bytes {
598    fn borrow(&self) -> &[u8] {
599        self.as_slice()
600    }
601}
602
603impl IntoIterator for Bytes {
604    type Item = u8;
605    type IntoIter = IntoIter<Bytes>;
606
607    fn into_iter(self) -> Self::IntoIter {
608        IntoIter::new(self)
609    }
610}
611
612impl<'a> IntoIterator for &'a Bytes {
613    type Item = &'a u8;
614    type IntoIter = core::slice::Iter<'a, u8>;
615
616    fn into_iter(self) -> Self::IntoIter {
617        self.as_slice().iter()
618    }
619}
620
621impl FromIterator<u8> for Bytes {
622    fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self {
623        Vec::from_iter(into_iter).into()
624    }
625}
626
627// impl Eq
628
629impl PartialEq for Bytes {
630    fn eq(&self, other: &Bytes) -> bool {
631        self.as_slice() == other.as_slice()
632    }
633}
634
635impl PartialOrd for Bytes {
636    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
637        self.as_slice().partial_cmp(other.as_slice())
638    }
639}
640
641impl Ord for Bytes {
642    fn cmp(&self, other: &Bytes) -> cmp::Ordering {
643        self.as_slice().cmp(other.as_slice())
644    }
645}
646
647impl Eq for Bytes {}
648
649impl PartialEq<[u8]> for Bytes {
650    fn eq(&self, other: &[u8]) -> bool {
651        self.as_slice() == other
652    }
653}
654
655impl PartialOrd<[u8]> for Bytes {
656    fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
657        self.as_slice().partial_cmp(other)
658    }
659}
660
661impl PartialEq<Bytes> for [u8] {
662    fn eq(&self, other: &Bytes) -> bool {
663        *other == *self
664    }
665}
666
667impl PartialOrd<Bytes> for [u8] {
668    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
669        <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
670    }
671}
672
673impl PartialEq<str> for Bytes {
674    fn eq(&self, other: &str) -> bool {
675        self.as_slice() == other.as_bytes()
676    }
677}
678
679impl PartialOrd<str> for Bytes {
680    fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
681        self.as_slice().partial_cmp(other.as_bytes())
682    }
683}
684
685impl PartialEq<Bytes> for str {
686    fn eq(&self, other: &Bytes) -> bool {
687        *other == *self
688    }
689}
690
691impl PartialOrd<Bytes> for str {
692    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
693        <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
694    }
695}
696
697impl PartialEq<Vec<u8>> for Bytes {
698    fn eq(&self, other: &Vec<u8>) -> bool {
699        *self == other[..]
700    }
701}
702
703impl PartialOrd<Vec<u8>> for Bytes {
704    fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
705        self.as_slice().partial_cmp(&other[..])
706    }
707}
708
709impl PartialEq<Bytes> for Vec<u8> {
710    fn eq(&self, other: &Bytes) -> bool {
711        *other == *self
712    }
713}
714
715impl PartialOrd<Bytes> for Vec<u8> {
716    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
717        <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
718    }
719}
720
721impl PartialEq<String> for Bytes {
722    fn eq(&self, other: &String) -> bool {
723        *self == other[..]
724    }
725}
726
727impl PartialOrd<String> for Bytes {
728    fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
729        self.as_slice().partial_cmp(other.as_bytes())
730    }
731}
732
733impl PartialEq<Bytes> for String {
734    fn eq(&self, other: &Bytes) -> bool {
735        *other == *self
736    }
737}
738
739impl PartialOrd<Bytes> for String {
740    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
741        <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
742    }
743}
744
745impl PartialEq<Bytes> for &[u8] {
746    fn eq(&self, other: &Bytes) -> bool {
747        *other == *self
748    }
749}
750
751impl PartialOrd<Bytes> for &[u8] {
752    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
753        <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
754    }
755}
756
757impl PartialEq<Bytes> for &str {
758    fn eq(&self, other: &Bytes) -> bool {
759        *other == *self
760    }
761}
762
763impl PartialOrd<Bytes> for &str {
764    fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
765        <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
766    }
767}
768
769impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes
770where
771    Bytes: PartialEq<T>,
772{
773    fn eq(&self, other: &&'a T) -> bool {
774        *self == **other
775    }
776}
777
778impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes
779where
780    Bytes: PartialOrd<T>,
781{
782    fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> {
783        self.partial_cmp(&**other)
784    }
785}
786
787// impl From
788
789impl Default for Bytes {
790    #[inline]
791    fn default() -> Bytes {
792        Bytes::new()
793    }
794}
795
796impl From<&'static [u8]> for Bytes {
797    fn from(slice: &'static [u8]) -> Bytes {
798        Bytes::from_static(slice)
799    }
800}
801
802impl From<&'static str> for Bytes {
803    fn from(slice: &'static str) -> Bytes {
804        Bytes::from_static(slice.as_bytes())
805    }
806}
807
808impl From<Vec<u8>> for Bytes {
809    fn from(vec: Vec<u8>) -> Bytes {
810        let slice = vec.into_boxed_slice();
811        slice.into()
812    }
813}
814
815impl From<Box<[u8]>> for Bytes {
816    fn from(slice: Box<[u8]>) -> Bytes {
817        // Box<[u8]> doesn't contain a heap allocation for empty slices,
818        // so the pointer isn't aligned enough for the KIND_VEC stashing to
819        // work.
820        if slice.is_empty() {
821            return Bytes::new();
822        }
823
824        let len = slice.len();
825        let ptr = Box::into_raw(slice) as *mut u8;
826
827        if ptr as usize & 0x1 == 0 {
828            let data = ptr_map(ptr, |addr| addr | KIND_VEC);
829            Bytes {
830                ptr,
831                len,
832                data: AtomicPtr::new(data.cast()),
833                vtable: &PROMOTABLE_EVEN_VTABLE,
834            }
835        } else {
836            Bytes {
837                ptr,
838                len,
839                data: AtomicPtr::new(ptr.cast()),
840                vtable: &PROMOTABLE_ODD_VTABLE,
841            }
842        }
843    }
844}
845
846impl From<String> for Bytes {
847    fn from(s: String) -> Bytes {
848        Bytes::from(s.into_bytes())
849    }
850}
851
852impl From<Bytes> for Vec<u8> {
853    fn from(bytes: Bytes) -> Vec<u8> {
854        let bytes = mem::ManuallyDrop::new(bytes);
855        unsafe { (bytes.vtable.to_vec)(&bytes.data, bytes.ptr, bytes.len) }
856    }
857}
858
859// ===== impl Vtable =====
860
861impl fmt::Debug for Vtable {
862    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
863        f.debug_struct("Vtable")
864            .field("clone", &(self.clone as *const ()))
865            .field("drop", &(self.drop as *const ()))
866            .finish()
867    }
868}
869
870// ===== impl StaticVtable =====
871
872const STATIC_VTABLE: Vtable = Vtable {
873    clone: static_clone,
874    to_vec: static_to_vec,
875    drop: static_drop,
876};
877
878unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
879    let slice = slice::from_raw_parts(ptr, len);
880    Bytes::from_static(slice)
881}
882
883unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
884    let slice = slice::from_raw_parts(ptr, len);
885    slice.to_vec()
886}
887
888unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) {
889    // nothing to drop for &'static [u8]
890}
891
892// ===== impl PromotableVtable =====
893
894static PROMOTABLE_EVEN_VTABLE: Vtable = Vtable {
895    clone: promotable_even_clone,
896    to_vec: promotable_even_to_vec,
897    drop: promotable_even_drop,
898};
899
900static PROMOTABLE_ODD_VTABLE: Vtable = Vtable {
901    clone: promotable_odd_clone,
902    to_vec: promotable_odd_to_vec,
903    drop: promotable_odd_drop,
904};
905
906unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
907    let shared = data.load(Ordering::Acquire);
908    let kind = shared as usize & KIND_MASK;
909
910    if kind == KIND_ARC {
911        shallow_clone_arc(shared.cast(), ptr, len)
912    } else {
913        debug_assert_eq!(kind, KIND_VEC);
914        let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
915        shallow_clone_vec(data, shared, buf, ptr, len)
916    }
917}
918
919unsafe fn promotable_to_vec(
920    data: &AtomicPtr<()>,
921    ptr: *const u8,
922    len: usize,
923    f: fn(*mut ()) -> *mut u8,
924) -> Vec<u8> {
925    let shared = data.load(Ordering::Acquire);
926    let kind = shared as usize & KIND_MASK;
927
928    if kind == KIND_ARC {
929        shared_to_vec_impl(shared.cast(), ptr, len)
930    } else {
931        // If Bytes holds a Vec, then the offset must be 0.
932        debug_assert_eq!(kind, KIND_VEC);
933
934        let buf = f(shared);
935
936        let cap = (ptr as usize - buf as usize) + len;
937
938        // Copy back buffer
939        ptr::copy(ptr, buf, len);
940
941        Vec::from_raw_parts(buf, len, cap)
942    }
943}
944
945unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
946    promotable_to_vec(data, ptr, len, |shared| {
947        ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
948    })
949}
950
951unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
952    data.with_mut(|shared| {
953        let shared = *shared;
954        let kind = shared as usize & KIND_MASK;
955
956        if kind == KIND_ARC {
957            release_shared(shared.cast());
958        } else {
959            debug_assert_eq!(kind, KIND_VEC);
960            let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
961            free_boxed_slice(buf, ptr, len);
962        }
963    });
964}
965
966unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
967    let shared = data.load(Ordering::Acquire);
968    let kind = shared as usize & KIND_MASK;
969
970    if kind == KIND_ARC {
971        shallow_clone_arc(shared as _, ptr, len)
972    } else {
973        debug_assert_eq!(kind, KIND_VEC);
974        shallow_clone_vec(data, shared, shared.cast(), ptr, len)
975    }
976}
977
978unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
979    promotable_to_vec(data, ptr, len, |shared| shared.cast())
980}
981
982unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
983    data.with_mut(|shared| {
984        let shared = *shared;
985        let kind = shared as usize & KIND_MASK;
986
987        if kind == KIND_ARC {
988            release_shared(shared.cast());
989        } else {
990            debug_assert_eq!(kind, KIND_VEC);
991
992            free_boxed_slice(shared.cast(), ptr, len);
993        }
994    });
995}
996
997unsafe fn free_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) {
998    let cap = (offset as usize - buf as usize) + len;
999    dealloc(buf, Layout::from_size_align(cap, 1).unwrap())
1000}
1001
1002// ===== impl SharedVtable =====
1003
1004struct Shared {
1005    // Holds arguments to dealloc upon Drop, but otherwise doesn't use them
1006    buf: *mut u8,
1007    cap: usize,
1008    ref_cnt: AtomicUsize,
1009}
1010
1011impl Drop for Shared {
1012    fn drop(&mut self) {
1013        unsafe { dealloc(self.buf, Layout::from_size_align(self.cap, 1).unwrap()) }
1014    }
1015}
1016
1017// Assert that the alignment of `Shared` is divisible by 2.
1018// This is a necessary invariant since we depend on allocating `Shared` a
1019// shared object to implicitly carry the `KIND_ARC` flag in its pointer.
1020// This flag is set when the LSB is 0.
1021const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2.
1022
1023static SHARED_VTABLE: Vtable = Vtable {
1024    clone: shared_clone,
1025    to_vec: shared_to_vec,
1026    drop: shared_drop,
1027};
1028
1029const KIND_ARC: usize = 0b0;
1030const KIND_VEC: usize = 0b1;
1031const KIND_MASK: usize = 0b1;
1032
1033unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1034    let shared = data.load(Ordering::Relaxed);
1035    shallow_clone_arc(shared as _, ptr, len)
1036}
1037
1038unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> Vec<u8> {
1039    // Check that the ref_cnt is 1 (unique).
1040    //
1041    // If it is unique, then it is set to 0 with AcqRel fence for the same
1042    // reason in release_shared.
1043    //
1044    // Otherwise, we take the other branch and call release_shared.
1045    if (*shared)
1046        .ref_cnt
1047        .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Relaxed)
1048        .is_ok()
1049    {
1050        let buf = (*shared).buf;
1051        let cap = (*shared).cap;
1052
1053        // Deallocate Shared
1054        drop(Box::from_raw(shared as *mut mem::ManuallyDrop<Shared>));
1055
1056        // Copy back buffer
1057        ptr::copy(ptr, buf, len);
1058
1059        Vec::from_raw_parts(buf, len, cap)
1060    } else {
1061        let v = slice::from_raw_parts(ptr, len).to_vec();
1062        release_shared(shared);
1063        v
1064    }
1065}
1066
1067unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
1068    shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len)
1069}
1070
1071unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
1072    data.with_mut(|shared| {
1073        release_shared(shared.cast());
1074    });
1075}
1076
1077unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {
1078    let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed);
1079
1080    if old_size > usize::MAX >> 1 {
1081        crate::abort();
1082    }
1083
1084    Bytes {
1085        ptr,
1086        len,
1087        data: AtomicPtr::new(shared as _),
1088        vtable: &SHARED_VTABLE,
1089    }
1090}
1091
1092#[cold]
1093unsafe fn shallow_clone_vec(
1094    atom: &AtomicPtr<()>,
1095    ptr: *const (),
1096    buf: *mut u8,
1097    offset: *const u8,
1098    len: usize,
1099) -> Bytes {
1100    // If  the buffer is still tracked in a `Vec<u8>`. It is time to
1101    // promote the vec to an `Arc`. This could potentially be called
1102    // concurrently, so some care must be taken.
1103
1104    // First, allocate a new `Shared` instance containing the
1105    // `Vec` fields. It's important to note that `ptr`, `len`,
1106    // and `cap` cannot be mutated without having `&mut self`.
1107    // This means that these fields will not be concurrently
1108    // updated and since the buffer hasn't been promoted to an
1109    // `Arc`, those three fields still are the components of the
1110    // vector.
1111    let shared = Box::new(Shared {
1112        buf,
1113        cap: (offset as usize - buf as usize) + len,
1114        // Initialize refcount to 2. One for this reference, and one
1115        // for the new clone that will be returned from
1116        // `shallow_clone`.
1117        ref_cnt: AtomicUsize::new(2),
1118    });
1119
1120    let shared = Box::into_raw(shared);
1121
1122    // The pointer should be aligned, so this assert should
1123    // always succeed.
1124    debug_assert!(
1125        0 == (shared as usize & KIND_MASK),
1126        "internal: Box<Shared> should have an aligned pointer",
1127    );
1128
1129    // Try compare & swapping the pointer into the `arc` field.
1130    // `Release` is used synchronize with other threads that
1131    // will load the `arc` field.
1132    //
1133    // If the `compare_exchange` fails, then the thread lost the
1134    // race to promote the buffer to shared. The `Acquire`
1135    // ordering will synchronize with the `compare_exchange`
1136    // that happened in the other thread and the `Shared`
1137    // pointed to by `actual` will be visible.
1138    match atom.compare_exchange(ptr as _, shared as _, Ordering::AcqRel, Ordering::Acquire) {
1139        Ok(actual) => {
1140            debug_assert!(actual as usize == ptr as usize);
1141            // The upgrade was successful, the new handle can be
1142            // returned.
1143            Bytes {
1144                ptr: offset,
1145                len,
1146                data: AtomicPtr::new(shared as _),
1147                vtable: &SHARED_VTABLE,
1148            }
1149        }
1150        Err(actual) => {
1151            // The upgrade failed, a concurrent clone happened. Release
1152            // the allocation that was made in this thread, it will not
1153            // be needed.
1154            let shared = Box::from_raw(shared);
1155            mem::forget(*shared);
1156
1157            // Buffer already promoted to shared storage, so increment ref
1158            // count.
1159            shallow_clone_arc(actual as _, offset, len)
1160        }
1161    }
1162}
1163
1164unsafe fn release_shared(ptr: *mut Shared) {
1165    // `Shared` storage... follow the drop steps from Arc.
1166    if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 {
1167        return;
1168    }
1169
1170    // This fence is needed to prevent reordering of use of the data and
1171    // deletion of the data.  Because it is marked `Release`, the decreasing
1172    // of the reference count synchronizes with this `Acquire` fence. This
1173    // means that use of the data happens before decreasing the reference
1174    // count, which happens before this fence, which happens before the
1175    // deletion of the data.
1176    //
1177    // As explained in the [Boost documentation][1],
1178    //
1179    // > It is important to enforce any possible access to the object in one
1180    // > thread (through an existing reference) to *happen before* deleting
1181    // > the object in a different thread. This is achieved by a "release"
1182    // > operation after dropping a reference (any access to the object
1183    // > through this reference must obviously happened before), and an
1184    // > "acquire" operation before deleting the object.
1185    //
1186    // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
1187    //
1188    // Thread sanitizer does not support atomic fences. Use an atomic load
1189    // instead.
1190    (*ptr).ref_cnt.load(Ordering::Acquire);
1191
1192    // Drop the data
1193    drop(Box::from_raw(ptr));
1194}
1195
1196// Ideally we would always use this version of `ptr_map` since it is strict
1197// provenance compatible, but it results in worse codegen. We will however still
1198// use it on miri because it gives better diagnostics for people who test bytes
1199// code with miri.
1200//
1201// See https://github.com/tokio-rs/bytes/pull/545 for more info.
1202#[cfg(miri)]
1203fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1204where
1205    F: FnOnce(usize) -> usize,
1206{
1207    let old_addr = ptr as usize;
1208    let new_addr = f(old_addr);
1209    let diff = new_addr.wrapping_sub(old_addr);
1210    ptr.wrapping_add(diff)
1211}
1212
1213#[cfg(not(miri))]
1214fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1215where
1216    F: FnOnce(usize) -> usize,
1217{
1218    let old_addr = ptr as usize;
1219    let new_addr = f(old_addr);
1220    new_addr as *mut u8
1221}
1222
1223// compile-fails
1224
1225/// ```compile_fail
1226/// use bytes::Bytes;
1227/// #[deny(unused_must_use)]
1228/// {
1229///     let mut b1 = Bytes::from("hello world");
1230///     b1.split_to(6);
1231/// }
1232/// ```
1233fn _split_to_must_use() {}
1234
1235/// ```compile_fail
1236/// use bytes::Bytes;
1237/// #[deny(unused_must_use)]
1238/// {
1239///     let mut b1 = Bytes::from("hello world");
1240///     b1.split_off(6);
1241/// }
1242/// ```
1243fn _split_off_must_use() {}
1244
1245// fuzz tests
1246#[cfg(all(test, loom))]
1247mod fuzz {
1248    use loom::sync::Arc;
1249    use loom::thread;
1250
1251    use super::Bytes;
1252    #[test]
1253    fn bytes_cloning_vec() {
1254        loom::model(|| {
1255            let a = Bytes::from(b"abcdefgh".to_vec());
1256            let addr = a.as_ptr() as usize;
1257
1258            // test the Bytes::clone is Sync by putting it in an Arc
1259            let a1 = Arc::new(a);
1260            let a2 = a1.clone();
1261
1262            let t1 = thread::spawn(move || {
1263                let b: Bytes = (*a1).clone();
1264                assert_eq!(b.as_ptr() as usize, addr);
1265            });
1266
1267            let t2 = thread::spawn(move || {
1268                let b: Bytes = (*a2).clone();
1269                assert_eq!(b.as_ptr() as usize, addr);
1270            });
1271
1272            t1.join().unwrap();
1273            t2.join().unwrap();
1274        });
1275    }
1276}
bytes/bytes.rs

bytes/
bytes.rs
