From 6af75492a94606a5c8e771e6150c095352532ee9 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 19 Aug 2021 10:12:49 -0400 Subject: [PATCH] Swap out iframely (#1706) * Replace Iframely. Fixes #1681 * Add post_link_tags to nginx * Adding post_link_tags route * Cleaning up post_link_tags * Changing PostLink to SiteMetadata, adding it to the API. * Fixing issue when local has no openssl certs. * Fixing an issue with pictrs errors * Revert "Fixing issue when local has no openssl certs." This reverts commit dbf7d1b1ee03846e5ef7b7156e618424f1150e1d. * Add ca-certs to dockerfile for volume mount. * Cleaning up fetch_pictrs request * Changing to fetch_site_data --- Cargo.lock | 531 +++++++++++++++++++++------ README.md | 1 - ansible/lemmy.yml | 4 - ansible/lemmy_dev.yml | 4 - ansible/templates/docker-compose.yml | 10 - ansible/templates/nginx.conf | 6 - config/config.hjson | 2 - crates/api/src/lib.rs | 3 + crates/api/src/post.rs | 19 +- crates/api_common/src/post.rs | 11 + crates/api_crud/src/post/create.rs | 9 +- crates/api_crud/src/post/update.rs | 9 +- crates/apub/src/objects/post.rs | 8 +- crates/utils/Cargo.toml | 1 + crates/utils/src/request.rs | 142 +++++-- crates/utils/src/settings/structs.rs | 2 - crates/websocket/src/lib.rs | 1 + docker/dev/docker-compose.yml | 10 - docker/dev/volume_mount.dockerfile | 2 +- docker/federation/docker-compose.yml | 7 - docker/federation/nginx.conf | 30 -- docker/iframely.config.local.js | 283 -------------- docker/lemmy.hjson | 2 + docker/prod/docker-compose.yml | 9 - src/api_routes.rs | 6 +- src/main.rs | 1 + 26 files changed, 585 insertions(+), 528 deletions(-) delete mode 100644 docker/iframely.config.local.js diff --git a/Cargo.lock b/Cargo.lock index d859f274..4cdcc47f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,16 +58,16 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.0-beta.8" +version = "3.0.0-beta.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd16d6b846983ffabfd081e1a67abd7698094fcbe7b3d9bcf1acbc6f546a516" +checksum = "01260589f1aafad11224002741eb37bc603b4ce55b4e3556d2b2122f9aac7c51" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-tls", "actix-utils", - "ahash 0.7.4", + "ahash", "base64 0.13.0", "bitflags", "bytes", @@ -91,7 +91,7 @@ dependencies = [ "rand 0.8.4", "regex", "serde", - "sha-1 0.9.6", + "sha-1 0.9.7", "smallvec", "time 0.2.27", "tokio", @@ -204,7 +204,7 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash 0.7.4", + "ahash", "bytes", "cfg-if", "cookie", @@ -281,12 +281,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "ahash" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" - [[package]] name = "ahash" version = "0.7.4" @@ -356,6 +350,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -469,9 +469,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitvec" @@ -540,9 +540,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytemuck" -version = "1.7.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966d2ab714d0f785dbac0a0396251a35280aeb42413281617d0209ab4898435" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" [[package]] name = "byteorder" @@ -587,9 +587,9 @@ checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" [[package]] name = "cc" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" [[package]] name = "cfg-if" @@ -629,6 +629,15 @@ dependencies = [ "chrono", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -722,9 +731,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -861,13 +870,14 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.14" +version = "0.99.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" +checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", "proc-macro2 1.0.28", "quote 1.0.9", + "rustc_version 0.3.3", "syn 1.0.74", ] @@ -1015,18 +1025,18 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fallible_collections" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad9169582543d2cfe9961be1e9eaf4fc42f9aa3483f7c485717b8dde36466ea" +checksum = "eaefd4190151d458f16f0793d3452d7f13aeb3701566a4cefc4c37598876cc00" dependencies = [ - "hashbrown 0.9.1", + "hashbrown", ] [[package]] name = "fastrand" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" +checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e" dependencies = [ "instant", ] @@ -1074,12 +1084,28 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" +[[package]] +name = "futf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.16" @@ -1134,7 +1160,7 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c54913bae956fb8df7f4dc6fc90362aa72e69148e3f39041fbe8742d21e0ac57" dependencies = [ - "autocfg", + "autocfg 1.0.1", "proc-macro-hack", "proc-macro2 1.0.28", "quote 1.0.9", @@ -1159,7 +1185,7 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78" dependencies = [ - "autocfg", + "autocfg 1.0.1", "futures-channel", "futures-core", "futures-io", @@ -1250,20 +1276,14 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" -dependencies = [ - "ahash 0.4.7", -] - [[package]] name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] [[package]] name = "heck" @@ -1300,6 +1320,20 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a164bb2ceaeff4f42542bdb847c41517c78a60f5649671b2a07312b6e117549" +[[package]] +name = "html5ever" +version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c213fa6a618dc1da552f54f85cba74b05d8e883c92ec4e89067736938084c26e" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "http" version = "0.2.4" @@ -1313,9 +1347,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" dependencies = [ "bytes", "http", @@ -1387,9 +1421,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.9" +version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" +checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11" dependencies = [ "bytes", "futures-channel", @@ -1464,15 +1498,15 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ - "autocfg", - "hashbrown 0.11.2", + "autocfg 1.0.1", + "hashbrown", ] [[package]] name = "instant" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" dependencies = [ "cfg-if", ] @@ -1509,9 +1543,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.51" +version = "0.3.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" +checksum = "ce791b7ca6638aae45be056e068fc756d871eb3b3b10b8efa62d1c9cec616752" dependencies = [ "wasm-bindgen", ] @@ -1885,6 +1919,7 @@ dependencies = [ "thiserror", "tokio", "url", + "webpage", ] [[package]] @@ -1936,9 +1971,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" [[package]] name = "local-channel" @@ -1988,12 +2023,34 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "markup5ever" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897636f9850c3eef4905a5540683ed53dc9393860f0846cab2c2ddf9939862ff" +dependencies = [ + "phf", + "phf_codegen", + "serde", + "serde_derive", + "serde_json", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -2002,15 +2059,15 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memoffset" @@ -2018,7 +2075,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -2064,7 +2121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -2091,9 +2148,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", @@ -2107,6 +2164,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + [[package]] name = "nom" version = "6.1.2" @@ -2134,7 +2197,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-integer", "num-traits", ] @@ -2145,7 +2208,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-traits", ] @@ -2155,7 +2218,7 @@ version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-integer", "num-traits", ] @@ -2166,7 +2229,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-integer", "num-traits", ] @@ -2177,7 +2240,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -2234,7 +2297,7 @@ version = "0.9.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1996d2d305e561b70d1ee0c53f1542833f4e1ac6ce9a6708b6ff2738ca67dc82" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cc", "libc", "pkg-config", @@ -2332,20 +2395,58 @@ dependencies = [ "sha-1 0.8.2", ] +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ "proc-macro2 1.0.28", "quote 1.0.9", @@ -2397,6 +2498,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -2478,6 +2585,25 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -2503,6 +2629,16 @@ dependencies = [ "rand_hc 0.3.1", ] +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2523,6 +2659,21 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -2541,6 +2692,15 @@ dependencies = [ "getrandom 0.2.3", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -2559,13 +2719,66 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rayon" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ - "autocfg", + "autocfg 1.0.1", "crossbeam-deque", "either", "rayon-core", @@ -2584,11 +2797,20 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ "bitflags", ] @@ -2695,7 +2917,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -2793,7 +3024,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -2802,6 +3042,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.127" @@ -2905,9 +3154,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" +checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" dependencies = [ "block-buffer 0.9.0", "cfg-if", @@ -2961,11 +3210,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + [[package]] name = "slab" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" [[package]] name = "smallvec" @@ -2986,9 +3241,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" dependencies = [ "libc", "winapi", @@ -3016,7 +3271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -3058,6 +3313,40 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "string_cache" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared", + "precomputed-hash", + "serde", + "string_cache_codegen", + "string_cache_shared", +] + +[[package]] +name = "string_cache_codegen" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2 1.0.28", + "quote 1.0.9", + "string_cache_shared", +] + +[[package]] +name = "string_cache_shared" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" + [[package]] name = "strsim" version = "0.9.3" @@ -3099,6 +3388,17 @@ dependencies = [ "unicode-xid 0.1.0", ] +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.74" @@ -3130,6 +3430,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "tendril" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -3221,9 +3532,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" dependencies = [ "tinyvec_macros", ] @@ -3240,7 +3551,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cf844b23c6131f624accf65ce0e4e9956a8bb329400ea5bcc26ae3a5c20b0b" dependencies = [ - "autocfg", + "autocfg 1.0.1", "bytes", "libc", "memchr", @@ -3315,9 +3626,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8" dependencies = [ "lazy_static", ] @@ -3379,12 +3690,9 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" [[package]] name = "unicode-bidi" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" -dependencies = [ - "matches", -] +checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" [[package]] name = "unicode-normalization" @@ -3438,6 +3746,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "0.8.2" @@ -3484,9 +3798,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.74" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" +checksum = "b608ecc8f4198fe8680e2ed18eccab5f0cd4caaf3d83516fa5fb2e927fda2586" dependencies = [ "cfg-if", "serde", @@ -3496,9 +3810,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.74" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" +checksum = "580aa3a91a63d23aac5b6b267e2d13cb4f363e31dce6c352fca4752ae12e479f" dependencies = [ "bumpalo", "lazy_static", @@ -3511,9 +3825,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.24" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" +checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" dependencies = [ "cfg-if", "js-sys", @@ -3523,9 +3837,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.74" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" +checksum = "171ebf0ed9e1458810dfcb31f2e766ad6b3a89dbda42d8901f2b268277e5f09c" dependencies = [ "quote 1.0.9", "wasm-bindgen-macro-support", @@ -3533,9 +3847,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.74" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" +checksum = "6c2657dd393f03aa2a659c25c6ae18a13a4048cebd220e147933ea837efc589f" dependencies = [ "proc-macro2 1.0.28", "quote 1.0.9", @@ -3546,20 +3860,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.74" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" +checksum = "2e0c4a743a309662d45f4ede961d7afa4ba4131a59a639f29b0069c3798bbcc2" [[package]] name = "web-sys" -version = "0.3.51" +version = "0.3.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" +checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpage" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adf8232428a373fe9dc80dc3b7ba15c35621746691869ae2aa4fe0bb6f01098" +dependencies = [ + "html5ever", + "serde", + "serde_json", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/README.md b/README.md index 0a08c61c..ade81cbf 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,6 @@ Each Lemmy server can set its own moderation policy; appointing site-wide admins - Can transfer site and communities to others. - Can fully erase your data, replacing all posts and comments. - NSFW post / community support. -- OEmbed support via Iframely. - High performance. - Server is written in rust. - Front end is `~80kB` gzipped. diff --git a/ansible/lemmy.yml b/ansible/lemmy.yml index 969d6aff..91bda571 100644 --- a/ansible/lemmy.yml +++ b/ansible/lemmy.yml @@ -72,16 +72,12 @@ - src: 'templates/nginx.conf' dest: '/etc/nginx/sites-enabled/lemmy.conf' mode: '0644' - - src: '../docker/iframely.config.local.js' - dest: '{{lemmy_base_dir}}/iframely.config.local.js' - mode: '0600' vars: lemmy_docker_image: "dessalines/lemmy:{{ lookup('file', 'VERSION') }}" lemmy_docker_ui_image: "dessalines/lemmy-ui:{{ lookup('file', 'VERSION') }}" lemmy_port: "8536" lemmy_ui_port: "1235" pictshare_port: "8537" - iframely_port: "8538" - name: add config file (only during initial setup) template: diff --git a/ansible/lemmy_dev.yml b/ansible/lemmy_dev.yml index 141900a3..a685cbe3 100644 --- a/ansible/lemmy_dev.yml +++ b/ansible/lemmy_dev.yml @@ -61,16 +61,12 @@ - src: 'templates/nginx.conf' dest: '/etc/nginx/sites-enabled/lemmy.conf' mode: '0644' - - src: '../docker/iframely.config.local.js' - dest: '{{lemmy_base_dir}}/iframely.config.local.js' - mode: '0600' vars: lemmy_docker_image: "dessalines/lemmy:dev" lemmy_docker_ui_image: "dessalines/lemmy-ui:{{ lookup('file', 'VERSION') }}" lemmy_port: "8536" lemmy_ui_port: "1235" pictshare_port: "8537" - iframely_port: "8538" postgres_password: "{{ lookup('password', 'passwords/{{ inventory_hostname }}/postgres chars=ascii_letters,digits') }}" - name: add config file (only during initial setup) diff --git a/ansible/templates/docker-compose.yml b/ansible/templates/docker-compose.yml index 1d2609e6..5b04c28d 100644 --- a/ansible/templates/docker-compose.yml +++ b/ansible/templates/docker-compose.yml @@ -13,7 +13,6 @@ services: depends_on: - postgres - pictrs - - iframely lemmy-ui: image: {{ lemmy_docker_ui_image }} @@ -47,15 +46,6 @@ services: restart: always mem_limit: 200m - iframely: - image: dogbin/iframely:latest - ports: - - "127.0.0.1:8061:80" - volumes: - - ./iframely.config.local.js:/iframely/config.local.js:ro - restart: always - mem_limit: 200m - postfix: image: mwader/postfix-relay environment: diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 9c924442..132821c7 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -101,12 +101,6 @@ server { return 301 /pictrs/image/$1; } - location /iframely/ { - proxy_pass http://0.0.0.0:8061/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } # Anonymize IP addresses diff --git a/config/config.hjson b/config/config.hjson index 236751f4..cb08dda4 100644 --- a/config/config.hjson +++ b/config/config.hjson @@ -37,8 +37,6 @@ jwt_secret: "changeme" # address where pictrs is available pictrs_url: "http://pictrs:8080" - # address where iframely is available - iframely_url: "http://iframely" # maximum length of local community and user names actor_name_max_length: 20 # rate limits for various user actions, by user ip diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 6234e756..1c194b31 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -121,6 +121,9 @@ pub async fn match_websocket_operation( UserOperation::ResolvePostReport => { do_websocket_operation::(context, id, op, data).await } + UserOperation::GetSiteMetadata => { + do_websocket_operation::(context, id, op, data).await + } // Comment ops UserOperation::MarkCommentAsRead => { diff --git a/crates/api/src/post.rs b/crates/api/src/post.rs index 3de02731..1cb6c587 100644 --- a/crates/api/src/post.rs +++ b/crates/api/src/post.rs @@ -23,7 +23,7 @@ use lemmy_apub::{ use lemmy_db_queries::{source::post::Post_, Crud, Likeable, Saveable}; use lemmy_db_schema::source::{moderator::*, post::*}; use lemmy_db_views::post_view::PostView; -use lemmy_utils::{ApiError, ConnectionId, LemmyError}; +use lemmy_utils::{request::fetch_site_metadata, ApiError, ConnectionId, LemmyError}; use lemmy_websocket::{send::send_post_ws_message, LemmyContext, UserOperation}; use std::convert::TryInto; @@ -285,3 +285,20 @@ impl Perform for SavePost { Ok(PostResponse { post_view }) } } + +#[async_trait::async_trait(?Send)] +impl Perform for GetSiteMetadata { + type Response = GetSiteMetadataResponse; + + async fn perform( + &self, + context: &Data, + _websocket_id: Option, + ) -> Result { + let data: &Self = self; + + let metadata = fetch_site_metadata(context.client(), &data.url).await?; + + Ok(GetSiteMetadataResponse { metadata }) + } +} diff --git a/crates/api_common/src/post.rs b/crates/api_common/src/post.rs index bb6b3624..5a83a0c5 100644 --- a/crates/api_common/src/post.rs +++ b/crates/api_common/src/post.rs @@ -8,6 +8,7 @@ use lemmy_db_views_actor::{ community_moderator_view::CommunityModeratorView, community_view::CommunityView, }; +use lemmy_utils::request::SiteMetadata; use serde::{Deserialize, Serialize}; use url::Url; @@ -148,3 +149,13 @@ pub struct ListPostReports { pub struct ListPostReportsResponse { pub posts: Vec, } + +#[derive(Deserialize, Debug)] +pub struct GetSiteMetadata { + pub url: Url, +} + +#[derive(Serialize, Clone, Debug)] +pub struct GetSiteMetadataResponse { + pub metadata: SiteMetadata, +} diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 28c73968..547c657c 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -20,7 +20,7 @@ use lemmy_apub::{ use lemmy_db_queries::{source::post::Post_, Crud, Likeable}; use lemmy_db_schema::source::post::*; use lemmy_utils::{ - request::fetch_iframely_and_pictrs_data, + request::fetch_site_data, utils::{check_slurs, check_slurs_opt, clean_url_params, is_valid_post_title}, ApiError, ConnectionId, @@ -49,11 +49,10 @@ impl PerformCrud for CreatePost { check_community_ban(local_user_view.person.id, data.community_id, context.pool()).await?; - // Fetch Iframely and pictrs cached image + // Fetch post links and pictrs cached image let data_url = data.url.as_ref(); - let (iframely_response, pictrs_thumbnail) = - fetch_iframely_and_pictrs_data(context.client(), data_url).await?; - let (embed_title, embed_description, embed_html) = iframely_response + let (metadata_res, pictrs_thumbnail) = fetch_site_data(context.client(), data_url).await; + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index 4b63ebd2..d52538e0 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -5,7 +5,7 @@ use lemmy_apub::activities::{post::create_or_update::CreateOrUpdatePost, CreateO use lemmy_db_queries::{source::post::Post_, Crud}; use lemmy_db_schema::{naive_now, source::post::*}; use lemmy_utils::{ - request::fetch_iframely_and_pictrs_data, + request::fetch_site_data, utils::{check_slurs_opt, clean_url_params, is_valid_post_title}, ApiError, ConnectionId, @@ -49,11 +49,10 @@ impl PerformCrud for EditPost { return Err(ApiError::err("no_post_edit_allowed").into()); } - // Fetch Iframely and Pictrs cached image + // Fetch post links and Pictrs cached image let data_url = data.url.as_ref(); - let (iframely_response, pictrs_thumbnail) = - fetch_iframely_and_pictrs_data(context.client(), data_url).await?; - let (embed_title, embed_description, embed_html) = iframely_response + let (metadata_res, pictrs_thumbnail) = fetch_site_data(context.client(), data_url).await; + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 43a019d1..4773f8c5 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -31,7 +31,7 @@ use lemmy_db_schema::{ }, }; use lemmy_utils::{ - request::fetch_iframely_and_pictrs_data, + request::fetch_site_data, utils::{check_slurs, convert_datetime, markdown_to_html, remove_slurs}, LemmyError, }; @@ -188,12 +188,12 @@ impl FromApub for Post { let community = extract_community(&page.to, context, request_counter).await?; let thumbnail_url: Option = page.image.clone().map(|i| i.url); - let (iframely_response, pictrs_thumbnail) = if let Some(url) = &page.url { - fetch_iframely_and_pictrs_data(context.client(), Some(url)).await? + let (metadata_res, pictrs_thumbnail) = if let Some(url) = &page.url { + fetch_site_data(context.client(), Some(url)).await } else { (None, thumbnail_url) }; - let (embed_title, embed_description, embed_html) = iframely_response + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 018ed52c..c46a4e0c 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -38,3 +38,4 @@ http = "0.2.4" jsonwebtoken = "7.2.0" deser-hjson = "1.0.2" smart-default = "0.6.0" +webpage = { version = "1.1", default-features = false, features = ["serde"] } diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index c59180c5..a881cf02 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -3,10 +3,11 @@ use anyhow::anyhow; use log::error; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use reqwest::Client; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::future::Future; use thiserror::Error; use url::Url; +use webpage::HTML; #[derive(Clone, Debug, Error)] #[error("Error sending request, {0}")] @@ -47,31 +48,61 @@ where response.expect("retry http request") } -#[derive(Deserialize, Debug)] -pub struct IframelyResponse { +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct SiteMetadata { pub title: Option, pub description: Option, - thumbnail_url: Option, + image: Option, pub html: Option, } -pub(crate) async fn fetch_iframely( - client: &Client, - url: &Url, -) -> Result { - if let Some(iframely_url) = Settings::get().iframely_url { - let fetch_url = format!("{}/oembed?url={}", iframely_url, url); +/// Fetches the post link html tags (like title, description, image, etc) +pub async fn fetch_site_metadata(client: &Client, url: &Url) -> Result { + let response = retry(|| client.get(url.as_str()).send()).await?; - let response = retry(|| client.get(&fetch_url).send()).await?; + let html = response + .text() + .await + .map_err(|e| RecvError(e.to_string()))?; - let res: IframelyResponse = response - .json() - .await - .map_err(|e| RecvError(e.to_string()))?; - Ok(res) - } else { - Err(anyhow!("Missing Iframely URL in config.").into()) - } + let tags = html_to_site_metadata(&html)?; + + Ok(tags) +} + +fn html_to_site_metadata(html: &str) -> Result { + let page = HTML::from_string(html.to_string(), None)?; + + let page_title = page.title; + let page_description = page.description; + + let og_description = page + .opengraph + .properties + .get("description") + .map(|t| t.to_string()); + let og_title = page + .opengraph + .properties + .get("title") + .map(|t| t.to_string()); + let og_image = page + .opengraph + .images + .get(0) + .map(|ogo| Url::parse(&ogo.url).ok()) + .flatten(); + + let title = og_title.or(page_title); + let description = og_description.or(page_description); + let image = og_image; + + Ok(SiteMetadata { + title, + description, + image, + html: None, + }) } #[derive(Deserialize, Debug, Clone)] @@ -89,7 +120,7 @@ pub(crate) struct PictrsFile { pub(crate) async fn fetch_pictrs( client: &Client, image_url: &Url, -) -> Result, LemmyError> { +) -> Result { if let Some(pictrs_url) = Settings::get().pictrs_url { is_image_content_type(client, image_url).await?; @@ -107,37 +138,44 @@ pub(crate) async fn fetch_pictrs( .map_err(|e| RecvError(e.to_string()))?; if response.msg == "ok" { - Ok(Some(response)) + Ok(response) } else { Err(anyhow!("{}", &response.msg).into()) } } else { - Ok(None) + Err(anyhow!("pictrs_url not set up in config").into()) } } -pub async fn fetch_iframely_and_pictrs_data( +/// Both are options, since the URL might be either an html page, or an image +/// Returns the SiteMetadata, and a Pictrs URL, if there is a picture associated +pub async fn fetch_site_data( client: &Client, url: Option<&Url>, -) -> Result<(Option, Option), LemmyError> { +) -> (Option, Option) { match &url { Some(url) => { - // Fetch iframely data - let iframely_res_option = fetch_iframely(client, url).await.ok(); + // Fetch metadata + // Ignore errors, since it may be an image, or not have the data. + // Warning, this may ignore SSL errors + let metadata_option = fetch_site_metadata(client, url).await.ok(); // Fetch pictrs thumbnail - let pictrs_hash = match &iframely_res_option { - Some(iframely_res) => match &iframely_res.thumbnail_url { - Some(iframely_thumbnail_url) => fetch_pictrs(client, iframely_thumbnail_url) - .await? + let pictrs_hash = match &metadata_option { + Some(metadata_res) => match &metadata_res.image { + // Metadata, with image + // Try to generate a small thumbnail if there's a full sized one from post-links + Some(metadata_image) => fetch_pictrs(client, metadata_image) + .await .map(|r| r.files[0].file.to_owned()), - // Try to generate a small thumbnail if iframely is not supported + // Metadata, but no image None => fetch_pictrs(client, url) - .await? + .await .map(|r| r.files[0].file.to_owned()), }, + // No metadata, try to fetch the URL as an image None => fetch_pictrs(client, url) - .await? + .await .map(|r| r.files[0].file.to_owned()), }; @@ -151,11 +189,12 @@ pub async fn fetch_iframely_and_pictrs_data( )) .ok() }) + .ok() .flatten(); - Ok((iframely_res_option, pictrs_thumbnail)) + (metadata_option, pictrs_thumbnail) } - None => Ok((None, None)), + None => (None, None), } } @@ -176,12 +215,35 @@ async fn is_image_content_type(client: &Client, test: &Url) -> Result<(), LemmyE #[cfg(test)] mod tests { + use crate::request::fetch_site_metadata; + use url::Url; + + use super::SiteMetadata; + // These helped with testing - // #[test] - // fn test_iframely() { - // let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await; - // assert!(res.is_ok()); - // } + #[actix_rt::test] + async fn test_site_metadata() { + let client = reqwest::Client::default(); + let sample_url = Url::parse("https://www.redspark.nu/en/peoples-war/district-leader-of-chand-led-cpn-arrested-in-bhojpur/").unwrap(); + let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap(); + assert_eq!( + SiteMetadata { + title: Some("District Leader Of Chand Led CPN Arrested In Bhojpur - Redspark".to_string()), + description: Some("BHOJPUR: A district leader of the outlawed Netra Bikram Chand alias Biplav-led outfit has been arrested. According to District Police".to_string()), + image: Some(Url::parse("https://www.redspark.nu/wp-content/uploads/2020/03/netra-bikram-chand-attends-program-1272019033653-1000x0-845x653-1.jpg").unwrap()), + html: None, + }, sample_res); + + let youtube_url = Url::parse("https://www.youtube.com/watch?v=IquO_TcMZIQ").unwrap(); + let youtube_res = fetch_site_metadata(&client, &youtube_url).await.unwrap(); + assert_eq!( + SiteMetadata { + title: Some("A Hard Look at Rent and Rent Seeking with Michael Hudson & Pepe Escobar".to_string()), + description: Some("An interactive discussion on wealth inequality and the “Great Game” on the control of natural resources.In this webinar organized jointly by the Henry George...".to_string()), + image: Some(Url::parse("https://i.ytimg.com/vi/IquO_TcMZIQ/maxresdefault.jpg").unwrap()), + html: None, + }, youtube_res); + } // #[test] // fn test_pictshare() { diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 46dde919..c9537fe3 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -29,8 +29,6 @@ pub struct Settings { #[default(None)] pub pictrs_url: Option, #[default(None)] - pub iframely_url: Option, - #[default(None)] pub additional_slurs: Option, #[default(20)] pub actor_name_max_length: usize, diff --git a/crates/websocket/src/lib.rs b/crates/websocket/src/lib.rs index b866a2a2..e0151403 100644 --- a/crates/websocket/src/lib.rs +++ b/crates/websocket/src/lib.rs @@ -125,6 +125,7 @@ pub enum UserOperation { CommunityJoin, ModJoin, ChangePassword, + GetSiteMetadata, } #[derive(EnumString, ToString, Debug, Clone)] diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index a9e580a4..bc42d456 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -14,7 +14,6 @@ services: depends_on: - pictrs - postgres - - iframely lemmy-ui: image: dessalines/lemmy-ui:dev @@ -49,12 +48,3 @@ services: volumes: - ./volumes/pictrs:/mnt restart: always - - iframely: - image: dogbin/iframely:latest - ports: - - "8061:80" - volumes: - - ../iframely.config.local.js:/iframely/config.local.js:ro - restart: always - mem_limit: 200m diff --git a/docker/dev/volume_mount.dockerfile b/docker/dev/volume_mount.dockerfile index ae902e93..0af2a37f 100644 --- a/docker/dev/volume_mount.dockerfile +++ b/docker/dev/volume_mount.dockerfile @@ -22,7 +22,7 @@ FROM ubuntu:20.10 # Install libpq for postgres RUN apt-get update -y -RUN apt-get install -y libpq-dev +RUN apt-get install -y libpq-dev ca-certificates # Copy resources COPY --from=rust /app/lemmy_server /app/lemmy diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index a9d8863d..c2f05c17 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -14,7 +14,6 @@ services: restart: on-failure depends_on: - pictrs - - iframely - lemmy-alpha-ui - lemmy-beta-ui - lemmy-gamma-ui @@ -174,9 +173,3 @@ services: - POSTGRES_DB=lemmy volumes: - ./volumes/postgres_epsilon:/var/lib/postgresql/data - - iframely: - image: dogbin/iframely:latest - volumes: - - ../iframely.config.local.js:/iframely/config.local.js:ro - restart: always diff --git a/docker/federation/nginx.conf b/docker/federation/nginx.conf index 8bf2ed7f..425371be 100644 --- a/docker/federation/nginx.conf +++ b/docker/federation/nginx.conf @@ -40,12 +40,6 @@ http { # Cuts off the trailing slash on URLs to make them valid rewrite ^(.+)/+$ $1 permanent; } - location /iframely/ { - proxy_pass http://iframely:80/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } upstream lemmy-beta { @@ -85,12 +79,6 @@ http { # Cuts off the trailing slash on URLs to make them valid rewrite ^(.+)/+$ $1 permanent; } - location /iframely/ { - proxy_pass http://iframely:80/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } upstream lemmy-gamma { @@ -130,12 +118,6 @@ http { # Cuts off the trailing slash on URLs to make them valid rewrite ^(.+)/+$ $1 permanent; } - location /iframely/ { - proxy_pass http://iframely:80/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } upstream lemmy-delta { @@ -175,12 +157,6 @@ http { # Cuts off the trailing slash on URLs to make them valid rewrite ^(.+)/+$ $1 permanent; } - location /iframely/ { - proxy_pass http://iframely:80/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } upstream lemmy-epsilon { @@ -220,11 +196,5 @@ http { # Cuts off the trailing slash on URLs to make them valid rewrite ^(.+)/+$ $1 permanent; } - location /iframely/ { - proxy_pass http://iframely:80/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } } } diff --git a/docker/iframely.config.local.js b/docker/iframely.config.local.js deleted file mode 100644 index 8e8bc21f..00000000 --- a/docker/iframely.config.local.js +++ /dev/null @@ -1,283 +0,0 @@ -(function() { - var config = { - - // Specify a path for custom plugins. Custom plugins will override core plugins. - // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder', - - DEBUG: false, - RICH_LOG_ENABLED: false, - - // For embeds that require render, baseAppUrl will be used as the host. - baseAppUrl: "http://yourdomain.com", - relativeStaticUrl: "/r", - - // Or just skip built-in renders altogether - SKIP_IFRAMELY_RENDERS: true, - - // For legacy reasons the response format of Iframely open-source is - // different by default as it does not group the links array by rel. - // In order to get the same grouped response as in Cloud API, - // add `&group=true` to your request to change response per request - // or set `GROUP_LINKS` in your config to `true` for a global change. - GROUP_LINKS: true, - - // Number of maximum redirects to follow before aborting the page - // request with `redirect loop` error. - MAX_REDIRECTS: 4, - - SKIP_OEMBED_RE_LIST: [ - // /^https?:\/\/yourdomain\.com\//, - ], - - /* - // Used to pass parameters to the generate functions when creating HTML elements - // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div - GENERATE_LINK_PARAMS: { - disableSizeWrapper: true - }, - */ - - port: 80, //can be overridden by PORT env var - host: '0.0.0.0', // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246 - //can be overridden by HOST env var - - // Optional SSL cert, if you serve under HTTPS. - /* - ssl: { - key: require('fs').readFileSync(__dirname + '/key.pem'), - cert: require('fs').readFileSync(__dirname + '/cert.pem'), - port: 443 - }, - */ - - /* - Supported cache engines: - - no-cache - no caching will be used. - - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache). - - redis - https://github.com/mranney/node_redis. - - memcached - https://github.com/3rd-Eden/node-memcached - */ - CACHE_ENGINE: 'node-cache', - CACHE_TTL: 0, // In seconds. - // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record - // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead - - /* - // Redis cache options. - REDIS_OPTIONS: { - host: '127.0.0.1', - port: 6379 - }, - */ - - /* - // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations - MEMCACHED_OPTIONS: { - locations: "127.0.0.1:11211" - } - */ - - /* - // Access-Control-Allow-Origin list. - allowedOrigins: [ - "*", - "http://another_domain.com" - ], - */ - - /* - // Uncomment to enable plugin testing framework. - tests: { - mongodb: 'mongodb://localhost:27017/iframely-tests', - single_test_timeout: 10 * 1000, - plugin_test_period: 2 * 60 * 60 * 1000, - relaunch_script_period: 5 * 60 * 1000 - }, - */ - - // If there's no response from remote server, the timeout will occur after - RESPONSE_TIMEOUT: 5 * 1000, //ms - - /* From v1.4.0, Iframely supports HTTP/2 by default. Disable it, if you'd rather not. - Alternatively, you can also disable per origin. See `proxy` option below. - */ - // DISABLE_HTTP2: true, - - // Customize API calls to oembed endpoints. - ADD_OEMBED_PARAMS: [{ - // Endpoint url regexp array. - re: [/^http:\/\/api\.instagram\.com\/oembed/], - // Custom get params object. - params: { - hidecaption: true - } - }, { - re: [/^https:\/\/www\.facebook\.com\/plugins\/page\/oembed\.json/i], - params: { - show_posts: 0, - show_facepile: 0, - maxwidth: 600 - } - }, { - // match i=user or i=moment or i=timeline to configure these types invidually - // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed - re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i], - params: { - limit: 1, - maxwidth: 600 - } - /* - }, { - // Facebook https://developers.facebook.com/docs/plugins/oembed-endpoints - re: [/^https:\/\/www\.facebook\.com\/plugins\/\w+\/oembed\.json/i], - params: { - // Skip script tag and fb-root div. - omitscript: true - } - */ - }], - - /* - // Configure use of HTTP proxies as needed. - // You don't have to specify all options per regex - just what you need to override - PROXY: [{ - re: [/^https?:\/\/www\.domain\.com/], - proxy_server: 'http://1.2.3.4:8080', - user_agent: 'CHANGE YOUR AGENT', - headers: { - // HTTP headers - // Overrides previous params if overlapped. - }, - request_options: { - // Refer to: https://github.com/request/request - // Overrides previous params if overlapped. - }, - disable_http2: true - }], - */ - - // Customize API calls to 3rd parties. At the very least - configure required keys. - providerOptions: { - locale: "en_US", // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. - // Will be added as highest priotity in accept-language header with each request. - // Plus is used in FB, YouTube and perhaps other plugins - "twitter": { - "max-width": 550, - "min-width": 250, - hide_media: false, - hide_thread: false, - omit_script: false, - center: false, - // dnt: true, - cache_ttl: 100 * 365 * 24 * 3600 // 100 Years. - }, - readability: { - enabled: false - // allowPTagDescription: true // to enable description fallback to first paragraph - }, - images: { - loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes - checkFavicon: false // if true, will verify all favicons - }, - tumblr: { - consumer_key: "INSERT YOUR VALUE" - // media_only: true // disables status embeds for images and videos - will return plain media - }, - google: { - // https://developers.google.com/maps/documentation/embed/guide#api_key - maps_key: "INSERT YOUR VALUE" - }, - - /* - // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo - camoProxy: { - camo_proxy_key: "INSERT YOUR VALUE", - camo_proxy_host: "INSERT YOUR VALUE" - // ssl_only: true // will only proxy non-ssl images - }, - */ - - // List of query parameters to add to YouTube and Vimeo frames - // Start it with leading "?". Or omit alltogether for default values - // API key is optional, youtube will work without it too. - // It is probably the same API key you use for Google Maps. - youtube: { - // api_key: "INSERT YOUR VALUE", - get_params: "?rel=0&showinfo=1" // https://developers.google.com/youtube/player_parameters - }, - vimeo: { - get_params: "?byline=0&badge=0" // https://developer.vimeo.com/player/embedding - }, - - /* - soundcloud: { - old_player: true // enables classic player - }, - giphy: { - media_only: true // disables branded player for gifs and returns just the image - } - */ - /* - bandcamp: { - get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/', - media: { - album: { - height: 472, - 'max-width': 700 - }, - track: { - height: 120, - 'max-width': 700 - } - } - } - */ - }, - - // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*" - // with it, you can define what parsers do when they run accross unknown publisher. - // If absent or empty, all generic media parsers will be disabled except for known domains - // More about format: https://iframely.com/docs/qa-format - - /* - WHITELIST_WILDCARD: { - "twitter": { - "player": "allow", - "photo": "deny" - }, - "oembed": { - "video": "allow", - "photo": "allow", - "rich": "deny", - "link": "deny" - }, - "og": { - "video": ["allow", "ssl", "responsive"] - }, - "iframely": { - "survey": "allow", - "reader": "allow", - "player": "allow", - "image": "allow" - }, - "html-meta": { - "video": ["allow", "responsive"], - "promo": "allow" - } - } - */ - - // Black-list any of the inappropriate domains. Iframely will return 417 - // At minimum, keep your localhosts blacklisted to avoid SSRF - BLACKLIST_DOMAINS_RE: [ - /^https?:\/\/127\.0\.0\.1/i, - /^https?:\/\/localhost/i, - - // And this is AWS metadata service - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html - /^https?:\/\/169\.254\.169\.254/ - ] - }; - - module.exports = config; -})(); diff --git a/docker/lemmy.hjson b/docker/lemmy.hjson index 8f117e30..53f0ff17 100644 --- a/docker/lemmy.hjson +++ b/docker/lemmy.hjson @@ -20,6 +20,8 @@ # json web token for authorization between server and client jwt_secret: "changeme" # settings related to the postgresql database + # address where pictrs is available + pictrs_url: "http://pictrs:8080" database: { # name of the postgres database for lemmy database: "lemmy" diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 807faa11..a68c4987 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -23,7 +23,6 @@ services: depends_on: - postgres - pictrs - - iframely lemmy-ui: image: dessalines/lemmy-ui:0.11.3 @@ -46,11 +45,3 @@ services: - ./volumes/pictrs:/mnt restart: always - iframely: - image: dogbin/iframely:latest - ports: - - "127.0.0.1:8061:80" - volumes: - - ./iframely.config.local.js:/iframely/config.local.js:ro - restart: always - mem_limit: 200m diff --git a/src/api_routes.rs b/src/api_routes.rs index f8e8f7e1..a3bf39fe 100644 --- a/src/api_routes.rs +++ b/src/api_routes.rs @@ -88,7 +88,11 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) { "/report/resolve", web::put().to(route_post::), ) - .route("/report/list", web::get().to(route_get::)), + .route("/report/list", web::get().to(route_get::)) + .route( + "/site_metadata", + web::get().to(route_get::), + ), ) // Comment .service( diff --git a/src/main.rs b/src/main.rs index 6b4ac369..504ccc80 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ async fn main() -> Result<(), LemmyError> { ); let activity_queue = create_activity_queue(); + let chat_server = ChatServer::startup( pool.clone(), rate_limiter.clone(), -- 2.44.1