proc_macro2/
parse.rs

Help
1use crate::fallback::{
2    self, is_ident_continue, is_ident_start, Group, LexError, Literal, Span, TokenStream,
3    TokenStreamBuilder,
4};
5use crate::{Delimiter, Punct, Spacing, TokenTree};
6use core::char;
7use core::str::{Bytes, CharIndices, Chars};
8
9#[derive(Copy, Clone, Eq, PartialEq)]
10pub(crate) struct Cursor<'a> {
11    pub rest: &'a str,
12    #[cfg(span_locations)]
13    pub off: u32,
14}
15
16impl<'a> Cursor<'a> {
17    pub fn advance(&self, bytes: usize) -> Cursor<'a> {
18        let (_front, rest) = self.rest.split_at(bytes);
19        Cursor {
20            rest,
21            #[cfg(span_locations)]
22            off: self.off + _front.chars().count() as u32,
23        }
24    }
25
26    pub fn starts_with(&self, s: &str) -> bool {
27        self.rest.starts_with(s)
28    }
29
30    pub fn starts_with_char(&self, ch: char) -> bool {
31        self.rest.starts_with(ch)
32    }
33
34    pub fn starts_with_fn<Pattern>(&self, f: Pattern) -> bool
35    where
36        Pattern: FnMut(char) -> bool,
37    {
38        self.rest.starts_with(f)
39    }
40
41    pub fn is_empty(&self) -> bool {
42        self.rest.is_empty()
43    }
44
45    fn len(&self) -> usize {
46        self.rest.len()
47    }
48
49    fn as_bytes(&self) -> &'a [u8] {
50        self.rest.as_bytes()
51    }
52
53    fn bytes(&self) -> Bytes<'a> {
54        self.rest.bytes()
55    }
56
57    fn chars(&self) -> Chars<'a> {
58        self.rest.chars()
59    }
60
61    fn char_indices(&self) -> CharIndices<'a> {
62        self.rest.char_indices()
63    }
64
65    fn parse(&self, tag: &str) -> Result<Cursor<'a>, Reject> {
66        if self.starts_with(tag) {
67            Ok(self.advance(tag.len()))
68        } else {
69            Err(Reject)
70        }
71    }
72}
73
74pub(crate) struct Reject;
75type PResult<'a, O> = Result<(Cursor<'a>, O), Reject>;
76
77fn skip_whitespace(input: Cursor) -> Cursor {
78    let mut s = input;
79
80    while !s.is_empty() {
81        let byte = s.as_bytes()[0];
82        if byte == b'/' {
83            if s.starts_with("//")
84                && (!s.starts_with("///") || s.starts_with("////"))
85                && !s.starts_with("//!")
86            {
87                let (cursor, _) = take_until_newline_or_eof(s);
88                s = cursor;
89                continue;
90            } else if s.starts_with("/**/") {
91                s = s.advance(4);
92                continue;
93            } else if s.starts_with("/*")
94                && (!s.starts_with("/**") || s.starts_with("/***"))
95                && !s.starts_with("/*!")
96            {
97                match block_comment(s) {
98                    Ok((rest, _)) => {
99                        s = rest;
100                        continue;
101                    }
102                    Err(Reject) => return s,
103                }
104            }
105        }
106        match byte {
107            b' ' | 0x09..=0x0d => {
108                s = s.advance(1);
109                continue;
110            }
111            b if b.is_ascii() => {}
112            _ => {
113                let ch = s.chars().next().unwrap();
114                if is_whitespace(ch) {
115                    s = s.advance(ch.len_utf8());
116                    continue;
117                }
118            }
119        }
120        return s;
121    }
122    s
123}
124
125fn block_comment(input: Cursor) -> PResult<&str> {
126    if !input.starts_with("/*") {
127        return Err(Reject);
128    }
129
130    let mut depth = 0usize;
131    let bytes = input.as_bytes();
132    let mut i = 0usize;
133    let upper = bytes.len() - 1;
134
135    while i < upper {
136        if bytes[i] == b'/' && bytes[i + 1] == b'*' {
137            depth += 1;
138            i += 1; // eat '*'
139        } else if bytes[i] == b'*' && bytes[i + 1] == b'/' {
140            depth -= 1;
141            if depth == 0 {
142                return Ok((input.advance(i + 2), &input.rest[..i + 2]));
143            }
144            i += 1; // eat '/'
145        }
146        i += 1;
147    }
148
149    Err(Reject)
150}
151
152fn is_whitespace(ch: char) -> bool {
153    // Rust treats left-to-right mark and right-to-left mark as whitespace
154    ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}'
155}
156
157fn word_break(input: Cursor) -> Result<Cursor, Reject> {
158    match input.chars().next() {
159        Some(ch) if is_ident_continue(ch) => Err(Reject),
160        Some(_) | None => Ok(input),
161    }
162}
163
164// Rustc's representation of a macro expansion error in expression position or
165// type position.
166const ERROR: &str = "(/*ERROR*/)";
167
168pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
169    let mut trees = TokenStreamBuilder::new();
170    let mut stack = Vec::new();
171
172    loop {
173        input = skip_whitespace(input);
174
175        if let Ok((rest, ())) = doc_comment(input, &mut trees) {
176            input = rest;
177            continue;
178        }
179
180        #[cfg(span_locations)]
181        let lo = input.off;
182
183        let first = match input.bytes().next() {
184            Some(first) => first,
185            None => match stack.last() {
186                None => return Ok(trees.build()),
187                #[cfg(span_locations)]
188                Some((lo, _frame)) => {
189                    return Err(LexError {
190                        span: Span { lo: *lo, hi: *lo },
191                    })
192                }
193                #[cfg(not(span_locations))]
194                Some(_frame) => return Err(LexError { span: Span {} }),
195            },
196        };
197
198        if let Some(open_delimiter) = match first {
199            b'(' if !input.starts_with(ERROR) => Some(Delimiter::Parenthesis),
200            b'[' => Some(Delimiter::Bracket),
201            b'{' => Some(Delimiter::Brace),
202            _ => None,
203        } {
204            input = input.advance(1);
205            let frame = (open_delimiter, trees);
206            #[cfg(span_locations)]
207            let frame = (lo, frame);
208            stack.push(frame);
209            trees = TokenStreamBuilder::new();
210        } else if let Some(close_delimiter) = match first {
211            b')' => Some(Delimiter::Parenthesis),
212            b']' => Some(Delimiter::Bracket),
213            b'}' => Some(Delimiter::Brace),
214            _ => None,
215        } {
216            let frame = match stack.pop() {
217                Some(frame) => frame,
218                None => return Err(lex_error(input)),
219            };
220            #[cfg(span_locations)]
221            let (lo, frame) = frame;
222            let (open_delimiter, outer) = frame;
223            if open_delimiter != close_delimiter {
224                return Err(lex_error(input));
225            }
226            input = input.advance(1);
227            let mut g = Group::new(open_delimiter, trees.build());
228            g.set_span(Span {
229                #[cfg(span_locations)]
230                lo,
231                #[cfg(span_locations)]
232                hi: input.off,
233            });
234            trees = outer;
235            trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g)));
236        } else {
237            let (rest, mut tt) = match leaf_token(input) {
238                Ok((rest, tt)) => (rest, tt),
239                Err(Reject) => return Err(lex_error(input)),
240            };
241            tt.set_span(crate::Span::_new_fallback(Span {
242                #[cfg(span_locations)]
243                lo,
244                #[cfg(span_locations)]
245                hi: rest.off,
246            }));
247            trees.push_token_from_parser(tt);
248            input = rest;
249        }
250    }
251}
252
253fn lex_error(cursor: Cursor) -> LexError {
254    #[cfg(not(span_locations))]
255    let _ = cursor;
256    LexError {
257        span: Span {
258            #[cfg(span_locations)]
259            lo: cursor.off,
260            #[cfg(span_locations)]
261            hi: cursor.off,
262        },
263    }
264}
265
266fn leaf_token(input: Cursor) -> PResult<TokenTree> {
267    if let Ok((input, l)) = literal(input) {
268        // must be parsed before ident
269        Ok((input, TokenTree::Literal(crate::Literal::_new_fallback(l))))
270    } else if let Ok((input, p)) = punct(input) {
271        Ok((input, TokenTree::Punct(p)))
272    } else if let Ok((input, i)) = ident(input) {
273        Ok((input, TokenTree::Ident(i)))
274    } else if input.starts_with(ERROR) {
275        let rest = input.advance(ERROR.len());
276        let repr = crate::Literal::_new_fallback(Literal::_new(ERROR.to_owned()));
277        Ok((rest, TokenTree::Literal(repr)))
278    } else {
279        Err(Reject)
280    }
281}
282
283fn ident(input: Cursor) -> PResult<crate::Ident> {
284    if [
285        "r\"", "r#\"", "r##", "b\"", "b\'", "br\"", "br#", "c\"", "cr\"", "cr#",
286    ]
287    .iter()
288    .any(|prefix| input.starts_with(prefix))
289    {
290        Err(Reject)
291    } else {
292        ident_any(input)
293    }
294}
295
296fn ident_any(input: Cursor) -> PResult<crate::Ident> {
297    let raw = input.starts_with("r#");
298    let rest = input.advance((raw as usize) << 1);
299
300    let (rest, sym) = ident_not_raw(rest)?;
301
302    if !raw {
303        let ident = crate::Ident::_new(crate::imp::Ident::new_unchecked(
304            sym,
305            fallback::Span::call_site(),
306        ));
307        return Ok((rest, ident));
308    }
309
310    match sym {
311        "_" | "super" | "self" | "Self" | "crate" => return Err(Reject),
312        _ => {}
313    }
314
315    let ident = crate::Ident::_new(crate::imp::Ident::new_raw_unchecked(
316        sym,
317        fallback::Span::call_site(),
318    ));
319    Ok((rest, ident))
320}
321
322fn ident_not_raw(input: Cursor) -> PResult<&str> {
323    let mut chars = input.char_indices();
324
325    match chars.next() {
326        Some((_, ch)) if is_ident_start(ch) => {}
327        _ => return Err(Reject),
328    }
329
330    let mut end = input.len();
331    for (i, ch) in chars {
332        if !is_ident_continue(ch) {
333            end = i;
334            break;
335        }
336    }
337
338    Ok((input.advance(end), &input.rest[..end]))
339}
340
341pub(crate) fn literal(input: Cursor) -> PResult<Literal> {
342    let rest = literal_nocapture(input)?;
343    let end = input.len() - rest.len();
344    Ok((rest, Literal::_new(input.rest[..end].to_string())))
345}
346
347fn literal_nocapture(input: Cursor) -> Result<Cursor, Reject> {
348    if let Ok(ok) = string(input) {
349        Ok(ok)
350    } else if let Ok(ok) = byte_string(input) {
351        Ok(ok)
352    } else if let Ok(ok) = c_string(input) {
353        Ok(ok)
354    } else if let Ok(ok) = byte(input) {
355        Ok(ok)
356    } else if let Ok(ok) = character(input) {
357        Ok(ok)
358    } else if let Ok(ok) = float(input) {
359        Ok(ok)
360    } else if let Ok(ok) = int(input) {
361        Ok(ok)
362    } else {
363        Err(Reject)
364    }
365}
366
367fn literal_suffix(input: Cursor) -> Cursor {
368    match ident_not_raw(input) {
369        Ok((input, _)) => input,
370        Err(Reject) => input,
371    }
372}
373
374fn string(input: Cursor) -> Result<Cursor, Reject> {
375    if let Ok(input) = input.parse("\"") {
376        cooked_string(input)
377    } else if let Ok(input) = input.parse("r") {
378        raw_string(input)
379    } else {
380        Err(Reject)
381    }
382}
383
384fn cooked_string(mut input: Cursor) -> Result<Cursor, Reject> {
385    let mut chars = input.char_indices();
386
387    while let Some((i, ch)) = chars.next() {
388        match ch {
389            '"' => {
390                let input = input.advance(i + 1);
391                return Ok(literal_suffix(input));
392            }
393            '\r' => match chars.next() {
394                Some((_, '\n')) => {}
395                _ => break,
396            },
397            '\\' => match chars.next() {
398                Some((_, 'x')) => {
399                    backslash_x_char(&mut chars)?;
400                }
401                Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0')) => {}
402                Some((_, 'u')) => {
403                    backslash_u(&mut chars)?;
404                }
405                Some((newline, ch @ ('\n' | '\r'))) => {
406                    input = input.advance(newline + 1);
407                    trailing_backslash(&mut input, ch as u8)?;
408                    chars = input.char_indices();
409                }
410                _ => break,
411            },
412            _ch => {}
413        }
414    }
415    Err(Reject)
416}
417
418fn raw_string(input: Cursor) -> Result<Cursor, Reject> {
419    let (input, delimiter) = delimiter_of_raw_string(input)?;
420    let mut bytes = input.bytes().enumerate();
421    while let Some((i, byte)) = bytes.next() {
422        match byte {
423            b'"' if input.rest[i + 1..].starts_with(delimiter) => {
424                let rest = input.advance(i + 1 + delimiter.len());
425                return Ok(literal_suffix(rest));
426            }
427            b'\r' => match bytes.next() {
428                Some((_, b'\n')) => {}
429                _ => break,
430            },
431            _ => {}
432        }
433    }
434    Err(Reject)
435}
436
437fn byte_string(input: Cursor) -> Result<Cursor, Reject> {
438    if let Ok(input) = input.parse("b\"") {
439        cooked_byte_string(input)
440    } else if let Ok(input) = input.parse("br") {
441        raw_byte_string(input)
442    } else {
443        Err(Reject)
444    }
445}
446
447fn cooked_byte_string(mut input: Cursor) -> Result<Cursor, Reject> {
448    let mut bytes = input.bytes().enumerate();
449    while let Some((offset, b)) = bytes.next() {
450        match b {
451            b'"' => {
452                let input = input.advance(offset + 1);
453                return Ok(literal_suffix(input));
454            }
455            b'\r' => match bytes.next() {
456                Some((_, b'\n')) => {}
457                _ => break,
458            },
459            b'\\' => match bytes.next() {
460                Some((_, b'x')) => {
461                    backslash_x_byte(&mut bytes)?;
462                }
463                Some((_, b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"')) => {}
464                Some((newline, b @ (b'\n' | b'\r'))) => {
465                    input = input.advance(newline + 1);
466                    trailing_backslash(&mut input, b)?;
467                    bytes = input.bytes().enumerate();
468                }
469                _ => break,
470            },
471            b if b.is_ascii() => {}
472            _ => break,
473        }
474    }
475    Err(Reject)
476}
477
478fn delimiter_of_raw_string(input: Cursor) -> PResult<&str> {
479    for (i, byte) in input.bytes().enumerate() {
480        match byte {
481            b'"' => {
482                if i > 255 {
483                    // https://github.com/rust-lang/rust/pull/95251
484                    return Err(Reject);
485                }
486                return Ok((input.advance(i + 1), &input.rest[..i]));
487            }
488            b'#' => {}
489            _ => break,
490        }
491    }
492    Err(Reject)
493}
494
495fn raw_byte_string(input: Cursor) -> Result<Cursor, Reject> {
496    let (input, delimiter) = delimiter_of_raw_string(input)?;
497    let mut bytes = input.bytes().enumerate();
498    while let Some((i, byte)) = bytes.next() {
499        match byte {
500            b'"' if input.rest[i + 1..].starts_with(delimiter) => {
501                let rest = input.advance(i + 1 + delimiter.len());
502                return Ok(literal_suffix(rest));
503            }
504            b'\r' => match bytes.next() {
505                Some((_, b'\n')) => {}
506                _ => break,
507            },
508            other => {
509                if !other.is_ascii() {
510                    break;
511                }
512            }
513        }
514    }
515    Err(Reject)
516}
517
518fn c_string(input: Cursor) -> Result<Cursor, Reject> {
519    if let Ok(input) = input.parse("c\"") {
520        cooked_c_string(input)
521    } else if let Ok(input) = input.parse("cr") {
522        raw_c_string(input)
523    } else {
524        Err(Reject)
525    }
526}
527
528fn raw_c_string(input: Cursor) -> Result<Cursor, Reject> {
529    let (input, delimiter) = delimiter_of_raw_string(input)?;
530    let mut bytes = input.bytes().enumerate();
531    while let Some((i, byte)) = bytes.next() {
532        match byte {
533            b'"' if input.rest[i + 1..].starts_with(delimiter) => {
534                let rest = input.advance(i + 1 + delimiter.len());
535                return Ok(literal_suffix(rest));
536            }
537            b'\r' => match bytes.next() {
538                Some((_, b'\n')) => {}
539                _ => break,
540            },
541            b'\0' => break,
542            _ => {}
543        }
544    }
545    Err(Reject)
546}
547
548fn cooked_c_string(mut input: Cursor) -> Result<Cursor, Reject> {
549    let mut chars = input.char_indices();
550
551    while let Some((i, ch)) = chars.next() {
552        match ch {
553            '"' => {
554                let input = input.advance(i + 1);
555                return Ok(literal_suffix(input));
556            }
557            '\r' => match chars.next() {
558                Some((_, '\n')) => {}
559                _ => break,
560            },
561            '\\' => match chars.next() {
562                Some((_, 'x')) => {
563                    backslash_x_nonzero(&mut chars)?;
564                }
565                Some((_, 'n' | 'r' | 't' | '\\' | '\'' | '"')) => {}
566                Some((_, 'u')) => {
567                    if backslash_u(&mut chars)? == '\0' {
568                        break;
569                    }
570                }
571                Some((newline, ch @ ('\n' | '\r'))) => {
572                    input = input.advance(newline + 1);
573                    trailing_backslash(&mut input, ch as u8)?;
574                    chars = input.char_indices();
575                }
576                _ => break,
577            },
578            '\0' => break,
579            _ch => {}
580        }
581    }
582    Err(Reject)
583}
584
585fn byte(input: Cursor) -> Result<Cursor, Reject> {
586    let input = input.parse("b'")?;
587    let mut bytes = input.bytes().enumerate();
588    let ok = match bytes.next().map(|(_, b)| b) {
589        Some(b'\\') => match bytes.next().map(|(_, b)| b) {
590            Some(b'x') => backslash_x_byte(&mut bytes).is_ok(),
591            Some(b'n' | b'r' | b't' | b'\\' | b'0' | b'\'' | b'"') => true,
592            _ => false,
593        },
594        b => b.is_some(),
595    };
596    if !ok {
597        return Err(Reject);
598    }
599    let (offset, _) = bytes.next().ok_or(Reject)?;
600    if !input.chars().as_str().is_char_boundary(offset) {
601        return Err(Reject);
602    }
603    let input = input.advance(offset).parse("'")?;
604    Ok(literal_suffix(input))
605}
606
607fn character(input: Cursor) -> Result<Cursor, Reject> {
608    let input = input.parse("'")?;
609    let mut chars = input.char_indices();
610    let ok = match chars.next().map(|(_, ch)| ch) {
611        Some('\\') => match chars.next().map(|(_, ch)| ch) {
612            Some('x') => backslash_x_char(&mut chars).is_ok(),
613            Some('u') => backslash_u(&mut chars).is_ok(),
614            Some('n' | 'r' | 't' | '\\' | '0' | '\'' | '"') => true,
615            _ => false,
616        },
617        ch => ch.is_some(),
618    };
619    if !ok {
620        return Err(Reject);
621    }
622    let (idx, _) = chars.next().ok_or(Reject)?;
623    let input = input.advance(idx).parse("'")?;
624    Ok(literal_suffix(input))
625}
626
627macro_rules! next_ch {
628    ($chars:ident @ $pat:pat) => {
629        match $chars.next() {
630            Some((_, ch)) => match ch {
631                $pat => ch,
632                _ => return Err(Reject),
633            },
634            None => return Err(Reject),
635        }
636    };
637}
638
639fn backslash_x_char<I>(chars: &mut I) -> Result<(), Reject>
640where
641    I: Iterator<Item = (usize, char)>,
642{
643    next_ch!(chars @ '0'..='7');
644    next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
645    Ok(())
646}
647
648fn backslash_x_byte<I>(chars: &mut I) -> Result<(), Reject>
649where
650    I: Iterator<Item = (usize, u8)>,
651{
652    next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
653    next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
654    Ok(())
655}
656
657fn backslash_x_nonzero<I>(chars: &mut I) -> Result<(), Reject>
658where
659    I: Iterator<Item = (usize, char)>,
660{
661    let first = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
662    let second = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
663    if first == '0' && second == '0' {
664        Err(Reject)
665    } else {
666        Ok(())
667    }
668}
669
670fn backslash_u<I>(chars: &mut I) -> Result<char, Reject>
671where
672    I: Iterator<Item = (usize, char)>,
673{
674    next_ch!(chars @ '{');
675    let mut value = 0;
676    let mut len = 0;
677    for (_, ch) in chars {
678        let digit = match ch {
679            '0'..='9' => ch as u8 - b'0',
680            'a'..='f' => 10 + ch as u8 - b'a',
681            'A'..='F' => 10 + ch as u8 - b'A',
682            '_' if len > 0 => continue,
683            '}' if len > 0 => return char::from_u32(value).ok_or(Reject),
684            _ => break,
685        };
686        if len == 6 {
687            break;
688        }
689        value *= 0x10;
690        value += u32::from(digit);
691        len += 1;
692    }
693    Err(Reject)
694}
695
696fn trailing_backslash(input: &mut Cursor, mut last: u8) -> Result<(), Reject> {
697    let mut whitespace = input.bytes().enumerate();
698    loop {
699        if last == b'\r' && whitespace.next().map_or(true, |(_, b)| b != b'\n') {
700            return Err(Reject);
701        }
702        match whitespace.next() {
703            Some((_, b @ (b' ' | b'\t' | b'\n' | b'\r'))) => {
704                last = b;
705            }
706            Some((offset, _)) => {
707                *input = input.advance(offset);
708                return Ok(());
709            }
710            None => return Err(Reject),
711        }
712    }
713}
714
715fn float(input: Cursor) -> Result<Cursor, Reject> {
716    let mut rest = float_digits(input)?;
717    if let Some(ch) = rest.chars().next() {
718        if is_ident_start(ch) {
719            rest = ident_not_raw(rest)?.0;
720        }
721    }
722    word_break(rest)
723}
724
725fn float_digits(input: Cursor) -> Result<Cursor, Reject> {
726    let mut chars = input.chars().peekable();
727    match chars.next() {
728        Some(ch) if '0' <= ch && ch <= '9' => {}
729        _ => return Err(Reject),
730    }
731
732    let mut len = 1;
733    let mut has_dot = false;
734    let mut has_exp = false;
735    while let Some(&ch) = chars.peek() {
736        match ch {
737            '0'..='9' | '_' => {
738                chars.next();
739                len += 1;
740            }
741            '.' => {
742                if has_dot {
743                    break;
744                }
745                chars.next();
746                if chars
747                    .peek()
748                    .map_or(false, |&ch| ch == '.' || is_ident_start(ch))
749                {
750                    return Err(Reject);
751                }
752                len += 1;
753                has_dot = true;
754            }
755            'e' | 'E' => {
756                chars.next();
757                len += 1;
758                has_exp = true;
759                break;
760            }
761            _ => break,
762        }
763    }
764
765    if !(has_dot || has_exp) {
766        return Err(Reject);
767    }
768
769    if has_exp {
770        let token_before_exp = if has_dot {
771            Ok(input.advance(len - 1))
772        } else {
773            Err(Reject)
774        };
775        let mut has_sign = false;
776        let mut has_exp_value = false;
777        while let Some(&ch) = chars.peek() {
778            match ch {
779                '+' | '-' => {
780                    if has_exp_value {
781                        break;
782                    }
783                    if has_sign {
784                        return token_before_exp;
785                    }
786                    chars.next();
787                    len += 1;
788                    has_sign = true;
789                }
790                '0'..='9' => {
791                    chars.next();
792                    len += 1;
793                    has_exp_value = true;
794                }
795                '_' => {
796                    chars.next();
797                    len += 1;
798                }
799                _ => break,
800            }
801        }
802        if !has_exp_value {
803            return token_before_exp;
804        }
805    }
806
807    Ok(input.advance(len))
808}
809
810fn int(input: Cursor) -> Result<Cursor, Reject> {
811    let mut rest = digits(input)?;
812    if let Some(ch) = rest.chars().next() {
813        if is_ident_start(ch) {
814            rest = ident_not_raw(rest)?.0;
815        }
816    }
817    word_break(rest)
818}
819
820fn digits(mut input: Cursor) -> Result<Cursor, Reject> {
821    let base = if input.starts_with("0x") {
822        input = input.advance(2);
823        16
824    } else if input.starts_with("0o") {
825        input = input.advance(2);
826        8
827    } else if input.starts_with("0b") {
828        input = input.advance(2);
829        2
830    } else {
831        10
832    };
833
834    let mut len = 0;
835    let mut empty = true;
836    for b in input.bytes() {
837        match b {
838            b'0'..=b'9' => {
839                let digit = (b - b'0') as u64;
840                if digit >= base {
841                    return Err(Reject);
842                }
843            }
844            b'a'..=b'f' => {
845                let digit = 10 + (b - b'a') as u64;
846                if digit >= base {
847                    break;
848                }
849            }
850            b'A'..=b'F' => {
851                let digit = 10 + (b - b'A') as u64;
852                if digit >= base {
853                    break;
854                }
855            }
856            b'_' => {
857                if empty && base == 10 {
858                    return Err(Reject);
859                }
860                len += 1;
861                continue;
862            }
863            _ => break,
864        };
865        len += 1;
866        empty = false;
867    }
868    if empty {
869        Err(Reject)
870    } else {
871        Ok(input.advance(len))
872    }
873}
874
875fn punct(input: Cursor) -> PResult<Punct> {
876    let (rest, ch) = punct_char(input)?;
877    if ch == '\'' {
878        if ident_any(rest)?.0.starts_with_char('\'') {
879            Err(Reject)
880        } else {
881            Ok((rest, Punct::new('\'', Spacing::Joint)))
882        }
883    } else {
884        let kind = match punct_char(rest) {
885            Ok(_) => Spacing::Joint,
886            Err(Reject) => Spacing::Alone,
887        };
888        Ok((rest, Punct::new(ch, kind)))
889    }
890}
891
892fn punct_char(input: Cursor) -> PResult<char> {
893    if input.starts_with("//") || input.starts_with("/*") {
894        // Do not accept `/` of a comment as a punct.
895        return Err(Reject);
896    }
897
898    let mut chars = input.chars();
899    let first = match chars.next() {
900        Some(ch) => ch,
901        None => {
902            return Err(Reject);
903        }
904    };
905    let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
906    if recognized.contains(first) {
907        Ok((input.advance(first.len_utf8()), first))
908    } else {
909        Err(Reject)
910    }
911}
912
913fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> {
914    #[cfg(span_locations)]
915    let lo = input.off;
916    let (rest, (comment, inner)) = doc_comment_contents(input)?;
917    let fallback_span = Span {
918        #[cfg(span_locations)]
919        lo,
920        #[cfg(span_locations)]
921        hi: rest.off,
922    };
923    let span = crate::Span::_new_fallback(fallback_span);
924
925    let mut scan_for_bare_cr = comment;
926    while let Some(cr) = scan_for_bare_cr.find('\r') {
927        let rest = &scan_for_bare_cr[cr + 1..];
928        if !rest.starts_with('\n') {
929            return Err(Reject);
930        }
931        scan_for_bare_cr = rest;
932    }
933
934    let mut pound = Punct::new('#', Spacing::Alone);
935    pound.set_span(span);
936    trees.push_token_from_parser(TokenTree::Punct(pound));
937
938    if inner {
939        let mut bang = Punct::new('!', Spacing::Alone);
940        bang.set_span(span);
941        trees.push_token_from_parser(TokenTree::Punct(bang));
942    }
943
944    let doc_ident = crate::Ident::_new(crate::imp::Ident::new_unchecked("doc", fallback_span));
945    let mut equal = Punct::new('=', Spacing::Alone);
946    equal.set_span(span);
947    let mut literal = crate::Literal::string(comment);
948    literal.set_span(span);
949    let mut bracketed = TokenStreamBuilder::with_capacity(3);
950    bracketed.push_token_from_parser(TokenTree::Ident(doc_ident));
951    bracketed.push_token_from_parser(TokenTree::Punct(equal));
952    bracketed.push_token_from_parser(TokenTree::Literal(literal));
953    let group = Group::new(Delimiter::Bracket, bracketed.build());
954    let mut group = crate::Group::_new_fallback(group);
955    group.set_span(span);
956    trees.push_token_from_parser(TokenTree::Group(group));
957
958    Ok((rest, ()))
959}
960
961fn doc_comment_contents(input: Cursor) -> PResult<(&str, bool)> {
962    if input.starts_with("//!") {
963        let input = input.advance(3);
964        let (input, s) = take_until_newline_or_eof(input);
965        Ok((input, (s, true)))
966    } else if input.starts_with("/*!") {
967        let (input, s) = block_comment(input)?;
968        Ok((input, (&s[3..s.len() - 2], true)))
969    } else if input.starts_with("///") {
970        let input = input.advance(3);
971        if input.starts_with_char('/') {
972            return Err(Reject);
973        }
974        let (input, s) = take_until_newline_or_eof(input);
975        Ok((input, (s, false)))
976    } else if input.starts_with("/**") && !input.rest[3..].starts_with('*') {
977        let (input, s) = block_comment(input)?;
978        Ok((input, (&s[3..s.len() - 2], false)))
979    } else {
980        Err(Reject)
981    }
982}
983
984fn take_until_newline_or_eof(input: Cursor) -> (Cursor, &str) {
985    let chars = input.char_indices();
986
987    for (i, ch) in chars {
988        if ch == '\n' {
989            return (input.advance(i), &input.rest[..i]);
990        } else if ch == '\r' && input.rest[i + 1..].starts_with('\n') {
991            return (input.advance(i + 1), &input.rest[..i]);
992        }
993    }
994
995    (input.advance(input.len()), input.rest)
996}
proc_macro2/parse.rs

proc_macro2/
parse.rs
