diff --git a/CHANGELOG.md b/CHANGELOG.md index bef3d32..4516d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ uses [Semantic Versioning](https://semver.org/) and release tags in the form ## [Unreleased] +- Added a distinct message for a Facebook post Facebook gates as 18+. Such a + post can be public and still be withheld from anyone not logged in, so the + old wording — "make sure it is public" — sent the reader to check the one + thing that was already fine. The gate is recognised by the route Facebook + names in the document; the sentence a reader sees is drawn by script and + never reaches the HTML. - Fixed every Facebook reel and share link failing outright. Facebook began answering 400 to a request that claims a browser User-Agent but omits the `Sec-Fetch-*` headers a browser always sends on a top-level navigation, so diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index d2e13dd..bb1b249 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -204,6 +204,7 @@ "genericNoVideo": "No video was found at this link. Check the URL or paste a direct .mp4 file link.", "embeddedVideo": "Embedded video (Web)", "facebookNoVideo": "Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.", + "facebookAgeRestricted": "Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.", "highQuality720": "HD 720p (High quality)", "standardQuality480": "SD 480p (Standard)", "invalidVideoUrl": "Enter a valid video URL using http or https.", diff --git a/lib/l10n/app_vi.arb b/lib/l10n/app_vi.arb index e8cee9c..c66c67d 100644 --- a/lib/l10n/app_vi.arb +++ b/lib/l10n/app_vi.arb @@ -168,6 +168,7 @@ "genericNoVideo": "Không tìm thấy video tại liên kết này. Hãy kiểm tra URL hoặc dán liên kết trực tiếp tới file .mp4.", "embeddedVideo": "Video nhúng (Web)", "facebookNoVideo": "Không lấy được nội dung từ bài Facebook này. Hãy đảm bảo bài viết ở chế độ công khai; bài riêng tư và nhóm kín cần đăng nhập.", + "facebookAgeRestricted": "Facebook giới hạn bài này cho người trên 18 tuổi và chỉ trả nội dung cho tài khoản đã đăng nhập. Bài vẫn có thể đang công khai mà vẫn bị giữ lại, nên chế độ hiển thị không phải là nguyên nhân.", "highQuality720": "HD 720p (Chất lượng cao)", "standardQuality480": "SD 480p (Tiêu chuẩn)", "invalidVideoUrl": "Vui lòng nhập URL video hợp lệ sử dụng http hoặc https.", diff --git a/lib/l10n/extraction_failure_text.dart b/lib/l10n/extraction_failure_text.dart index feefd84..43dcf03 100644 --- a/lib/l10n/extraction_failure_text.dart +++ b/lib/l10n/extraction_failure_text.dart @@ -19,6 +19,7 @@ String describeExtractionFailure( detail ?? '', ), ExtractionFailureKind.facebookNoVideo => l10n.facebookNoVideo, + ExtractionFailureKind.facebookAgeRestricted => l10n.facebookAgeRestricted, ExtractionFailureKind.genericNoVideo => l10n.genericNoVideo, ExtractionFailureKind.instagramInvalidPost => l10n.instagramInvalidPost, ExtractionFailureKind.instagramLoginRequired => l10n.instagramLoginRequired, diff --git a/lib/l10n/generated/app_localizations.dart b/lib/l10n/generated/app_localizations.dart index db17e2f..56e3c8d 100644 --- a/lib/l10n/generated/app_localizations.dart +++ b/lib/l10n/generated/app_localizations.dart @@ -1106,6 +1106,12 @@ abstract class AppLocalizations { /// **'Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.'** String get facebookNoVideo; + /// No description provided for @facebookAgeRestricted. + /// + /// In en, this message translates to: + /// **'Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.'** + String get facebookAgeRestricted; + /// No description provided for @highQuality720. /// /// In en, this message translates to: diff --git a/lib/l10n/generated/app_localizations_en.dart b/lib/l10n/generated/app_localizations_en.dart index 261b0e2..bb2ebb7 100644 --- a/lib/l10n/generated/app_localizations_en.dart +++ b/lib/l10n/generated/app_localizations_en.dart @@ -619,6 +619,10 @@ class AppLocalizationsEn extends AppLocalizations { String get facebookNoVideo => 'Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.'; + @override + String get facebookAgeRestricted => + 'Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.'; + @override String get highQuality720 => 'HD 720p (High quality)'; diff --git a/lib/l10n/generated/app_localizations_vi.dart b/lib/l10n/generated/app_localizations_vi.dart index da9180b..a91a38d 100644 --- a/lib/l10n/generated/app_localizations_vi.dart +++ b/lib/l10n/generated/app_localizations_vi.dart @@ -617,6 +617,10 @@ class AppLocalizationsVi extends AppLocalizations { String get facebookNoVideo => 'Không lấy được nội dung từ bài Facebook này. Hãy đảm bảo bài viết ở chế độ công khai; bài riêng tư và nhóm kín cần đăng nhập.'; + @override + String get facebookAgeRestricted => + 'Facebook giới hạn bài này cho người trên 18 tuổi và chỉ trả nội dung cho tài khoản đã đăng nhập. Bài vẫn có thể đang công khai mà vẫn bị giữ lại, nên chế độ hiển thị không phải là nguyên nhân.'; + @override String get highQuality720 => 'HD 720p (Chất lượng cao)'; diff --git a/lib/services/extractors/extraction_failure.dart b/lib/services/extractors/extraction_failure.dart index 45bd8a8..a0e65b8 100644 --- a/lib/services/extractors/extraction_failure.dart +++ b/lib/services/extractors/extraction_failure.dart @@ -12,6 +12,12 @@ enum ExtractionFailureKind { linkAccessFailed, facebookNoVideo, + + /// Facebook gates the post as 18+, which no anonymous request can pass. + /// Distinct from [facebookNoVideo] because the post may well be public, and + /// telling the user to check that it is wastes their time. + facebookAgeRestricted, + genericNoVideo, instagramInvalidPost, instagramLoginRequired, diff --git a/lib/services/extractors/facebook_extractor.dart b/lib/services/extractors/facebook_extractor.dart index e8e8299..1680047 100644 --- a/lib/services/extractors/facebook_extractor.dart +++ b/lib/services/extractors/facebook_extractor.dart @@ -62,6 +62,12 @@ class FacebookExtractor extends BaseVideoExtractor { static final RegExp _stickerBucket = RegExp(r'/t39\.1997-'); static final RegExp _imageExtension = RegExp(r'\.(?:jpe?g|png|webp|gif)$'); + // Facebook names the route it served rather than the reason a reader sees: + // the "18+" sentence is drawn by script and never reaches the document. + // Measured across a public post, a watch page and the logged-out home page, + // none of which mention it, against an age-gated reel, which does 16 times. + static final RegExp _ageGate = RegExp('CometAgeInappropriate'); + // The Share action emits `/share//` and `/share/p//` for a // post. Neither redirects to the permalink for an anonymous request, so the // share link itself is what reaches the gallery fallback. `/share/r/` and @@ -92,6 +98,10 @@ class FacebookExtractor extends BaseVideoExtractor { } VideoMetadata? imageFallback; + // Any strategy may be the one that lands on the gate, so the answer is + // remembered across all three rather than read off the last page tried. + var ageGated = false; + void markAgeGated() => ageGated = true; VideoMetadata? accept(VideoMetadata? candidate) { if (candidate == null) return null; @@ -112,7 +122,12 @@ class FacebookExtractor extends BaseVideoExtractor { } // Strategy 1: the watch page itself. - var result = await _fromPage(cleanUrl, cleanUrl, cached: resolvedPage); + var result = await _fromPage( + cleanUrl, + cleanUrl, + cached: resolvedPage, + onAgeGate: markAgeGated, + ); final pageVideo = accept(result); if (pageVideo != null) { return _withPostPhotoFallback(pageVideo, cleanUrl); @@ -123,7 +138,7 @@ class FacebookExtractor extends BaseVideoExtractor { // interstitial than the full watch page. final embedUrl = 'https://www.facebook.com/plugins/video.php?href=${Uri.encodeComponent(cleanUrl)}'; - result = await _fromPage(embedUrl, cleanUrl); + result = await _fromPage(embedUrl, cleanUrl, onAgeGate: markAgeGated); final embedVideo = accept(result); if (embedVideo != null) { return _withPostPhotoFallback(embedVideo, cleanUrl); @@ -137,19 +152,30 @@ class FacebookExtractor extends BaseVideoExtractor { result = await _fromPage( mobileUrl, cleanUrl, - userAgent: AppConstants.mobileUserAgent, + onAgeGate: markAgeGated, ); final mobileVideo = accept(result); if (mobileVideo != null) { return _withPostPhotoFallback(mobileVideo, cleanUrl); } + // An age gate withholds the media however public the post is, so saying + // "make sure it is public" would send the reader to check the one thing + // that is already fine. + final failure = ExtractionFailure( + ageGated + ? ExtractionFailureKind.facebookAgeRestricted + : ExtractionFailureKind.facebookNoVideo, + ); + if (imageFallback != null) { if (_isKnownVideoLink(cleanUrl)) { throw ExtractionException( - const ExtractionFailure(ExtractionFailureKind.facebookNoVideo), - diagnosticCode: 'facebook_video_not_exposed', + failure, + diagnosticCode: ageGated + ? 'facebook_age_gated' + : 'facebook_video_not_exposed', attemptedStrategies: const ['page', 'embed', 'mobile'], ); } @@ -157,8 +183,10 @@ class FacebookExtractor extends BaseVideoExtractor { } throw ExtractionException( - const ExtractionFailure(ExtractionFailureKind.facebookNoVideo), - diagnosticCode: 'facebook_no_public_media', + failure, + diagnosticCode: ageGated + ? 'facebook_age_gated' + : 'facebook_no_public_media', attemptedStrategies: const ['page', 'embed', 'mobile'], ); } @@ -316,6 +344,7 @@ class FacebookExtractor extends BaseVideoExtractor { String originalUrl, { String userAgent = AppConstants.defaultUserAgent, http.Response? cached, + void Function()? onAgeGate, }) async { final String html; try { @@ -327,6 +356,8 @@ class FacebookExtractor extends BaseVideoExtractor { return null; } + if (_ageGate.hasMatch(html)) onAgeGate?.call(); + final hdUrl = _firstMatch(html, _hdPatterns); final sdUrl = _firstMatch(html, _sdPatterns); final openGraphVideo = _pageParser.openGraphVideo(html); diff --git a/test/extractor_fixture_test.dart b/test/extractor_fixture_test.dart index 1d727cc..2a34ac7 100644 --- a/test/extractor_fixture_test.dart +++ b/test/extractor_fixture_test.dart @@ -200,6 +200,33 @@ void main() { expect(result.qualities, hasLength(2)); }); + // A reel can be public and still serve an anonymous visitor nothing, because + // Facebook gates it as 18+. Reporting that as "make sure the post is public" + // sends the user to check the one thing that is already fine. The route name + // is the only signal in the document — the sentence a reader sees is drawn + // by script and never appears in the HTML. + test('Facebook tells an age-gated reel apart from a private one', () async { + ExtractorHttp.getOverride = (_, _) async => http.Response( + '', + 200, + ); + + await expectLater( + const FacebookExtractor().extract( + 'https://www.facebook.com/reel/1831368848220203/', + ), + throwsA( + isA().having( + (error) => error.failure.kind, + 'failure kind', + ExtractionFailureKind.facebookAgeRestricted, + ), + ), + ); + }); + test( 'Facebook reads an Open Graph video from a share landing page', () async {