Netdoc: allow parsing multiple routerdescs from a string.

tor-netdoc/src/lib.rs

@@ -40,5 +40,20 @@ pub mod routerdesc;
 pub mod version;
 
 pub use err::{Error, Pos};
+
 /// Alias for the Result type returned by most objects in this module.
 pub type Result<T> = std::result::Result<T, Error>;
+
+/// Indicates whether we should parse an annotated list of objects or a
+/// non-annotated list.
+#[derive(PartialEq, Debug)]
+pub enum AllowAnnotations {
+    /// Parsing a document where items might be annotated.
+    ///
+    /// Annotations are a list of zero or more items with keywords
+    /// beginning with @ that precede the items that are actually part
+    /// of the document.
+    AnnotationsAllowed,
+    /// Parsing a document where annotations are not allowed.
+    AnnotationsNotAllowed,
+}

tor-netdoc/src/microdesc.rs

@@ -13,9 +13,9 @@ use crate::family::RelayFamily;
 use crate::keyword::Keyword;
 use crate::parse::SectionRules;
 use crate::policy::PortPolicy;
-use crate::tokenize::NetDocReader;
+use crate::tokenize::{ItemResult, NetDocReader};
 use crate::util;
-use crate::{Error, Result};
+use crate::{AllowAnnotations, Error, Result};
 use tor_llcrypto::d;
 use tor_llcrypto::pk::{curve25519, ed25519, rsa};
 
@@ -106,10 +106,7 @@ impl MicrodescAnnotation {
     ) -> Result<MicrodescAnnotation> {
         use MicrodescKW::*;
 
-        let mut items = reader.pause_at(|item| match item {
-            Err(_) => false,
-            Ok(item) => !item.get_kwd().is_annotation(),
-        });
+        let mut items = reader.pause_at(|item| item.is_ok_with_non_annotation());
 
         let body = MICRODESC_ANNOTATIONS.parse(&mut items)?;
 
@@ -127,6 +124,7 @@ impl Microdesc {
     pub fn parse(s: &str) -> Result<Microdesc> {
         let mut items = crate::tokenize::NetDocReader::new(s);
         Self::parse_from_reader(&mut items).map_err(|e| e.within(s))
+        // TODO: must enforce that there are no more tokens in the reader.
     }
 
     /// Extract a single microdescriptor from a NetDocReader.
@@ -265,22 +263,6 @@ pub struct MicrodescReader<'a> {
     reader: NetDocReader<'a, MicrodescKW>,
 }
 
-/// Indicates whether we should parse an annotated list of votes or a
-/// non-annotated list.
-///
-/// TODO: Move this.
-#[derive(PartialEq, Debug)]
-pub enum AllowAnnotations {
-    /// Parsing a document where items might be annotated.
-    ///
-    /// Annotations are a list of zero or more items with keywords
-    /// beginning with @ that precede the items that are actually part
-    /// of the document.
-    AnnotationsAllowed,
-    /// Parsing a document where annotations are not allowed.
-    AnnotationsNotAllowed,
-}
-
 impl<'a> MicrodescReader<'a> {
     /// Construct a MicrodescReader to take microdescriptors from a string
     /// 's'.

tor-netdoc/src/routerdesc.rs

@@ -35,7 +35,7 @@ use crate::parse::{Section, SectionRules};
 use crate::policy::*;
 use crate::tokenize::{ItemResult, NetDocReader};
 use crate::version::TorVersion;
-use crate::{Error, Result};
+use crate::{AllowAnnotations, Error, Result};
 
 use lazy_static::lazy_static;
 use std::{net, time};
@@ -44,6 +44,21 @@ use tor_llcrypto::pk::rsa::RSAIdentity;
 
 use digest::Digest;
 
+/// A router descriptor, with possible annotations.
+#[allow(dead_code)]
+pub struct AnnotatedRouterDesc {
+    ann: RouterAnnotation,
+    router: RouterDesc,
+}
+
+/// Annotations about a router descriptor, as stored on disc.
+#[allow(dead_code)] // don't warn about fields not getting read.
+pub struct RouterAnnotation {
+    source: Option<String>,
+    downloaded: Option<time::SystemTime>,
+    purpose: Option<String>,
+}
+
 /// Information about a relay, parsed from a router descriptor.
 ///
 /// This type does not hold all the information in the router descriptor
@@ -113,8 +128,9 @@ decl_keyword! {
     /// RouterKW is an instance of Keyword, used to denote the different
     /// Items that are recognized as appearing in a router descriptor.
     RouterKW {
-        annotation "@source" => A_SOURCE,
-        annotation "@downloaded-at" => A_DOWNLOADED_AT,
+        annotation "@source" => ANN_SOURCE,
+        annotation "@downloaded-at" => ANN_DOWNLOADED_AT,
+        annotation "@purpose" => ANN_PURPOSE,
         "accept" | "reject" => POLICY,
         "bandwidth" => BANDWIDTH,
         "bridge-distribution-request" => BRIDGE_DISTRIBUTION_REQUEST,
@@ -149,6 +165,17 @@ decl_keyword! {
 }
 
 lazy_static! {
+    /// Rules for parsing a set of router annotations.
+    static ref ROUTER_ANNOTATIONS : SectionRules<RouterKW> = {
+        use RouterKW::*;
+
+        let mut rules = SectionRules::new();
+        rules.add(ANN_SOURCE.rule());
+        rules.add(ANN_DOWNLOADED_AT.rule().args(1..));
+        rules.add(ANN_PURPOSE.rule().args(1..));
+        rules.add(ANN_UNRECOGNIZED.rule().may_repeat().obj_optional());
+        rules
+    };
     /// Rules for tokens that are allowed in the first part of a
     /// router descriptor.
     static ref ROUTER_HEADER_RULES : SectionRules<RouterKW> = {
@@ -211,6 +238,37 @@ lazy_static! {
     };
 }
 
+impl Default for RouterAnnotation {
+    fn default() -> Self {
+        RouterAnnotation {
+            source: None,
+            downloaded: None,
+            purpose: None,
+        }
+    }
+}
+
+impl RouterAnnotation {
+    fn take_from_reader(reader: &mut NetDocReader<'_, RouterKW>) -> Result<RouterAnnotation> {
+        use RouterKW::*;
+        let mut items = reader.pause_at(|item| item.is_ok_with_non_annotation());
+
+        let body = ROUTER_ANNOTATIONS.parse(&mut items)?;
+
+        let source = body.maybe(ANN_SOURCE).args_as_str().map(String::from);
+        let purpose = body.maybe(ANN_PURPOSE).args_as_str().map(String::from);
+        let downloaded = body
+            .maybe(ANN_DOWNLOADED_AT)
+            .parse_args_as_str::<ISO8601TimeSp>()?
+            .map(|t| t.into());
+        Ok(RouterAnnotation {
+            source,
+            purpose,
+            downloaded,
+        })
+    }
+}
+
 impl RouterDesc {
     /// Return true iff this router descriptor is expired.
     ///
@@ -241,7 +299,8 @@ impl RouterDesc {
         let body = ROUTER_BODY_RULES.parse(&mut reader)?;
 
         // Parse the signature.
-        let mut reader = reader.remaining();
+        let mut reader =
+            reader.new_pred(|item| item.is_ok_with_annotation() || item.is_ok_with_kwd(ROUTER));
         let sig = ROUTER_SIG_RULES.parse(&mut reader)?;
 
         Ok((header, body, sig))
@@ -262,7 +321,9 @@ impl RouterDesc {
     /// future work for now, partially because of limitations in the
     /// ed25519 API.
     pub fn parse(s: &str) -> Result<RouterDesc> {
-        Self::parse_internal(s).map_err(|e| e.within(s))
+        let mut reader = crate::tokenize::NetDocReader::new(s);
+        Self::parse_internal(&mut reader).map_err(|e| e.within(s))
+        // TODO: must enforce that there are no more tokens in the reader.
     }
 
     /// Helper: parse a router descriptor from `s`.
@@ -270,13 +331,13 @@ impl RouterDesc {
     /// This function does the same as parse(), but returns errors based on
     /// byte-wise positions.  The parse() function converts such errors
     /// into line-and-byte positions.
-    fn parse_internal(s: &str) -> Result<RouterDesc> {
+    fn parse_internal(r: &mut NetDocReader<'_, RouterKW>) -> Result<RouterDesc> {
         // TODO: This function is too long!  The little "paragraphs" here
         // that parse one item at a time should be made into sub-functions.
         use RouterKW::*;
 
-        let mut r = NetDocReader::new(s);
-        let (header, body, sig) = RouterDesc::parse_sections(&mut r)?;
+        let s = r.str();
+        let (header, body, sig) = RouterDesc::parse_sections(r)?;
 
         let start_offset = header.get_required(ROUTER)?.offset_in(s).unwrap();
 
@@ -522,6 +583,83 @@ impl RouterDesc {
     }
 }
 
+/// An iterator that parses one or more (possibly annotated
+/// router descriptors from a string.
+//
+// TODO: This is largely copy-pasted from MicrodescReader. Can/should they
+// be merged?
+pub struct RouterReader<'a> {
+    annotated: bool,
+    reader: NetDocReader<'a, RouterKW>,
+}
+
+fn advance_to_next_routerdesc(reader: &mut NetDocReader<'_, RouterKW>, annotated: bool) {
+    use RouterKW::*;
+    let iter = reader.iter();
+    loop {
+        let item = iter.peek();
+        match item {
+            Some(Ok(t)) => {
+                let kwd = t.get_kwd();
+                if (annotated && kwd.is_annotation()) || kwd == ROUTER {
+                    return;
+                }
+            }
+            Some(Err(_)) => {
+                // Skip over broken tokens.
+            }
+            None => {
+                return;
+            }
+        }
+        let _ = iter.next();
+    }
+}
+
+impl<'a> RouterReader<'a> {
+    /// Construct a RouterReader to take router descriptors from a string.
+    pub fn new(s: &'a str, allow: AllowAnnotations) -> Self {
+        let reader = NetDocReader::new(s);
+        let annotated = allow == AllowAnnotations::AnnotationsAllowed;
+        RouterReader { annotated, reader }
+    }
+
+    fn take_annotation(&mut self) -> Result<RouterAnnotation> {
+        if self.annotated {
+            RouterAnnotation::take_from_reader(&mut self.reader)
+        } else {
+            Ok(RouterAnnotation::default())
+        }
+    }
+
+    fn take_annotated_routerdesc_raw(&mut self) -> Result<AnnotatedRouterDesc> {
+        let ann = self.take_annotation()?;
+        let router = RouterDesc::parse_internal(&mut self.reader)?;
+        Ok(AnnotatedRouterDesc { router, ann })
+    }
+
+    fn take_annotated_routerdesc(&mut self) -> Result<AnnotatedRouterDesc> {
+        let result = self.take_annotated_routerdesc_raw();
+        if result.is_err() {
+            advance_to_next_routerdesc(&mut self.reader, self.annotated);
+        }
+        result
+    }
+}
+
+impl<'a> Iterator for RouterReader<'a> {
+    type Item = Result<AnnotatedRouterDesc>;
+    fn next(&mut self) -> Option<Self::Item> {
+        // Is there a next token? If not, we're done.
+        self.reader.iter().peek()?;
+
+        Some(
+            self.take_annotated_routerdesc()
+                .map_err(|e| e.within(self.reader.str())),
+        )
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

tor-netdoc/src/tokenize.rs

@@ -490,6 +490,10 @@ impl<'a, 'b, K: Keyword> MaybeItem<'a, 'b, K> {
 }
 
 pub trait ItemResult<K: Keyword> {
+    /// Return true if this is an ok result with an annotation.
+    fn is_ok_with_annotation(&self) -> bool;
+    /// Return true if this is an ok result with a non-annotation.
+    fn is_ok_with_non_annotation(&self) -> bool;
     /// Return true if this is an ok result with the keyword 'k'
     fn is_ok_with_kwd(&self, k: K) -> bool {
         self.is_ok_with_kwd_in(&[k])
@@ -501,6 +505,18 @@ pub trait ItemResult<K: Keyword> {
 }
 
 impl<'a, K: Keyword> ItemResult<K> for Result<Item<'a, K>> {
+    fn is_ok_with_annotation(&self) -> bool {
+        match self {
+            Ok(item) => item.get_kwd().is_annotation(),
+            Err(_) => false,
+        }
+    }
+    fn is_ok_with_non_annotation(&self) -> bool {
+        match self {
+            Ok(item) => !item.get_kwd().is_annotation(),
+            Err(_) => false,
+        }
+    }
     fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool {
         match self {
             Ok(item) => item.has_kwd_in(ks),

tor-netdoc/src/util.rs

@@ -34,6 +34,7 @@ impl<'a, I: Iterator, F: FnMut(&I::Item) -> bool> PauseAt<'a, I, F> {
     {
         PauseAt::from_peekable(self.peek, pred)
     }
+    #[allow(unused)]
     pub fn remaining(self) -> &'a mut Peekable<I> {
         self.peek
     }