fedi/lemmy
1
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-11-22 06:36:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2007 from LemmyNet/another_tokenizer_fix

Browse source 
Trying another tokenizer fix. #1964
This commit is contained in:
Dessalines 2021-12-21 17:24:44 -05:00 committed by GitHub
parent 9f64872d5a ad4715c2a3
commit 3065154f4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
crates/utils/src/request.rs
View file

@ -5,7 +5,7 @@ use reqwest_middleware::ClientWithMiddleware;
use serde::{Deserialize, Serialize};
use std::future::Future;
use thiserror::Error;
use tracing::error;
use tracing::{error, info};
use url::Url;
use webpage::HTML;
@ -64,12 +64,18 @@ pub async fn fetch_site_metadata(
client: &ClientWithMiddleware,
url: &Url,
) -> Result<SiteMetadata, LemmyError> {
info!("Fetching site metadata for url: {}", url);
let response = client.get(url.as_str()).send().await?;
let html = response
.text()
// Can't use .text() here, because it only checks the content header, not the actual bytes
// https://github.com/LemmyNet/lemmy/issues/1964
let html_bytes = response
.bytes()
.await
.map_err(|e| RecvError(e.to_string()))?;
.map_err(|e| RecvError(e.to_string()))?
.to_vec();
let html = String::from_utf8_lossy(&html_bytes);
let tags = html_to_site_metadata(&html)?;
@ -77,6 +83,20 @@ pub async fn fetch_site_metadata(
}
fn html_to_site_metadata(html: &str) -> Result<SiteMetadata, LemmyError> {
// Make sure the first line is doctype html
let first_line = html
.lines()
.into_iter()
.next()
.ok_or_else(|| LemmyError::from_message("No lines in html"))?
.to_lowercase();
if !first_line.starts_with("<!doctype html>") {
return Err(LemmyError::from_message(
"Site metadata page fetch is not DOCTYPE html",
));
}
let page = HTML::from_string(html.to_string(), None)?;
let page_title = page.title;