diff --git a/tokenizers/src/lib.rs b/tokenizers/src/lib.rs index 657d810ba..dd253e7f8 100644 --- a/tokenizers/src/lib.rs +++ b/tokenizers/src/lib.rs @@ -30,10 +30,11 @@ //! fn main() -> Result<()> { //! # #[cfg(feature = "http")] //! # { -//! let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?; +//! // needs http feature enabled +//! let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?; //! -//! let encoding = tokenizer.encode("Hey there!", false)?; -//! println!("{:?}", encoding.get_tokens()); +//! let encoding = tokenizer.encode("Hey there!", false)?; +//! println!("{:?}", encoding.get_tokens()); //! # } //! Ok(()) //! }