Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ uses [Semantic Versioning](https://semver.org/) and release tags in the form

## [Unreleased]

- Added a distinct message for a Facebook post Facebook gates as 18+. Such a
post can be public and still be withheld from anyone not logged in, so the
old wording — "make sure it is public" — sent the reader to check the one
thing that was already fine. The gate is recognised by the route Facebook
names in the document; the sentence a reader sees is drawn by script and
never reaches the HTML.
- Fixed every Facebook reel and share link failing outright. Facebook began
answering 400 to a request that claims a browser User-Agent but omits the
`Sec-Fetch-*` headers a browser always sends on a top-level navigation, so
Expand Down
1 change: 1 addition & 0 deletions lib/l10n/app_en.arb
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@
"genericNoVideo": "No video was found at this link. Check the URL or paste a direct .mp4 file link.",
"embeddedVideo": "Embedded video (Web)",
"facebookNoVideo": "Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.",
"facebookAgeRestricted": "Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.",
"highQuality720": "HD 720p (High quality)",
"standardQuality480": "SD 480p (Standard)",
"invalidVideoUrl": "Enter a valid video URL using http or https.",
Expand Down
1 change: 1 addition & 0 deletions lib/l10n/app_vi.arb
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@
"genericNoVideo": "Không tìm thấy video tại liên kết này. Hãy kiểm tra URL hoặc dán liên kết trực tiếp tới file .mp4.",
"embeddedVideo": "Video nhúng (Web)",
"facebookNoVideo": "Không lấy được nội dung từ bài Facebook này. Hãy đảm bảo bài viết ở chế độ công khai; bài riêng tư và nhóm kín cần đăng nhập.",
"facebookAgeRestricted": "Facebook giới hạn bài này cho người trên 18 tuổi và chỉ trả nội dung cho tài khoản đã đăng nhập. Bài vẫn có thể đang công khai mà vẫn bị giữ lại, nên chế độ hiển thị không phải là nguyên nhân.",
"highQuality720": "HD 720p (Chất lượng cao)",
"standardQuality480": "SD 480p (Tiêu chuẩn)",
"invalidVideoUrl": "Vui lòng nhập URL video hợp lệ sử dụng http hoặc https.",
Expand Down
1 change: 1 addition & 0 deletions lib/l10n/extraction_failure_text.dart
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ String describeExtractionFailure(
detail ?? '',
),
ExtractionFailureKind.facebookNoVideo => l10n.facebookNoVideo,
ExtractionFailureKind.facebookAgeRestricted => l10n.facebookAgeRestricted,
ExtractionFailureKind.genericNoVideo => l10n.genericNoVideo,
ExtractionFailureKind.instagramInvalidPost => l10n.instagramInvalidPost,
ExtractionFailureKind.instagramLoginRequired => l10n.instagramLoginRequired,
Expand Down
6 changes: 6 additions & 0 deletions lib/l10n/generated/app_localizations.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1106,6 +1106,12 @@ abstract class AppLocalizations {
/// **'Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.'**
String get facebookNoVideo;

/// No description provided for @facebookAgeRestricted.
///
/// In en, this message translates to:
/// **'Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.'**
String get facebookAgeRestricted;

/// No description provided for @highQuality720.
///
/// In en, this message translates to:
Expand Down
4 changes: 4 additions & 0 deletions lib/l10n/generated/app_localizations_en.dart
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,10 @@ class AppLocalizationsEn extends AppLocalizations {
String get facebookNoVideo =>
'Could not extract media from this Facebook post. Make sure it is public; private posts and closed groups require a login.';

@override
String get facebookAgeRestricted =>
'Facebook restricts this post to viewers over 18 and serves it only to someone logged in. The post can be public and still be withheld, so its visibility is not the problem.';

@override
String get highQuality720 => 'HD 720p (High quality)';

Expand Down
4 changes: 4 additions & 0 deletions lib/l10n/generated/app_localizations_vi.dart
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,10 @@ class AppLocalizationsVi extends AppLocalizations {
String get facebookNoVideo =>
'Không lấy được nội dung từ bài Facebook này. Hãy đảm bảo bài viết ở chế độ công khai; bài riêng tư và nhóm kín cần đăng nhập.';

@override
String get facebookAgeRestricted =>
'Facebook giới hạn bài này cho người trên 18 tuổi và chỉ trả nội dung cho tài khoản đã đăng nhập. Bài vẫn có thể đang công khai mà vẫn bị giữ lại, nên chế độ hiển thị không phải là nguyên nhân.';

@override
String get highQuality720 => 'HD 720p (Chất lượng cao)';

Expand Down
6 changes: 6 additions & 0 deletions lib/services/extractors/extraction_failure.dart
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ enum ExtractionFailureKind {
linkAccessFailed,

facebookNoVideo,

/// Facebook gates the post as 18+, which no anonymous request can pass.
/// Distinct from [facebookNoVideo] because the post may well be public, and
/// telling the user to check that it is wastes their time.
facebookAgeRestricted,

genericNoVideo,
instagramInvalidPost,
instagramLoginRequired,
Expand Down
45 changes: 38 additions & 7 deletions lib/services/extractors/facebook_extractor.dart
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ class FacebookExtractor extends BaseVideoExtractor {
static final RegExp _stickerBucket = RegExp(r'/t39\.1997-');
static final RegExp _imageExtension = RegExp(r'\.(?:jpe?g|png|webp|gif)$');

// Facebook names the route it served rather than the reason a reader sees:
// the "18+" sentence is drawn by script and never reaches the document.
// Measured across a public post, a watch page and the logged-out home page,
// none of which mention it, against an age-gated reel, which does 16 times.
static final RegExp _ageGate = RegExp('CometAgeInappropriate');

// The Share action emits `/share/<token>/` and `/share/p/<token>/` for a
// post. Neither redirects to the permalink for an anonymous request, so the
// share link itself is what reaches the gallery fallback. `/share/r/` and
Expand Down Expand Up @@ -92,6 +98,10 @@ class FacebookExtractor extends BaseVideoExtractor {
}

VideoMetadata? imageFallback;
// Any strategy may be the one that lands on the gate, so the answer is
// remembered across all three rather than read off the last page tried.
var ageGated = false;
void markAgeGated() => ageGated = true;

VideoMetadata? accept(VideoMetadata? candidate) {
if (candidate == null) return null;
Expand All @@ -112,7 +122,12 @@ class FacebookExtractor extends BaseVideoExtractor {
}

// Strategy 1: the watch page itself.
var result = await _fromPage(cleanUrl, cleanUrl, cached: resolvedPage);
var result = await _fromPage(
cleanUrl,
cleanUrl,
cached: resolvedPage,
onAgeGate: markAgeGated,
);
final pageVideo = accept(result);
if (pageVideo != null) {
return _withPostPhotoFallback(pageVideo, cleanUrl);
Expand All @@ -123,7 +138,7 @@ class FacebookExtractor extends BaseVideoExtractor {
// interstitial than the full watch page.
final embedUrl =
'https://www.facebook.com/plugins/video.php?href=${Uri.encodeComponent(cleanUrl)}';
result = await _fromPage(embedUrl, cleanUrl);
result = await _fromPage(embedUrl, cleanUrl, onAgeGate: markAgeGated);
final embedVideo = accept(result);
if (embedVideo != null) {
return _withPostPhotoFallback(embedVideo, cleanUrl);
Expand All @@ -137,28 +152,41 @@ class FacebookExtractor extends BaseVideoExtractor {
result = await _fromPage(
mobileUrl,
cleanUrl,

userAgent: AppConstants.mobileUserAgent,
onAgeGate: markAgeGated,
);
final mobileVideo = accept(result);
if (mobileVideo != null) {
return _withPostPhotoFallback(mobileVideo, cleanUrl);
}

// An age gate withholds the media however public the post is, so saying
// "make sure it is public" would send the reader to check the one thing
// that is already fine.
final failure = ExtractionFailure(
ageGated
? ExtractionFailureKind.facebookAgeRestricted
: ExtractionFailureKind.facebookNoVideo,
);

if (imageFallback != null) {
if (_isKnownVideoLink(cleanUrl)) {
throw ExtractionException(
const ExtractionFailure(ExtractionFailureKind.facebookNoVideo),
diagnosticCode: 'facebook_video_not_exposed',
failure,
diagnosticCode: ageGated
? 'facebook_age_gated'
: 'facebook_video_not_exposed',
attemptedStrategies: const ['page', 'embed', 'mobile'],
);
}
return _withPostPhotoFallback(imageFallback!, cleanUrl);
}

throw ExtractionException(
const ExtractionFailure(ExtractionFailureKind.facebookNoVideo),
diagnosticCode: 'facebook_no_public_media',
failure,
diagnosticCode: ageGated
? 'facebook_age_gated'
: 'facebook_no_public_media',
attemptedStrategies: const ['page', 'embed', 'mobile'],
);
}
Expand Down Expand Up @@ -316,6 +344,7 @@ class FacebookExtractor extends BaseVideoExtractor {
String originalUrl, {
String userAgent = AppConstants.defaultUserAgent,
http.Response? cached,
void Function()? onAgeGate,
}) async {
final String html;
try {
Expand All @@ -327,6 +356,8 @@ class FacebookExtractor extends BaseVideoExtractor {
return null;
}

if (_ageGate.hasMatch(html)) onAgeGate?.call();

final hdUrl = _firstMatch(html, _hdPatterns);
final sdUrl = _firstMatch(html, _sdPatterns);
final openGraphVideo = _pageParser.openGraphVideo(html);
Expand Down
27 changes: 27 additions & 0 deletions test/extractor_fixture_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,33 @@ void main() {
expect(result.qualities, hasLength(2));
});

// A reel can be public and still serve an anonymous visitor nothing, because
// Facebook gates it as 18+. Reporting that as "make sure the post is public"
// sends the user to check the one thing that is already fine. The route name
// is the only signal in the document — the sentence a reader sees is drawn
// by script and never appears in the HTML.
test('Facebook tells an age-gated reel apart from a private one', () async {
ExtractorHttp.getOverride = (_, _) async => http.Response(
'<html><body><script>'
'{"__crn":"comet.fbweb.CometAgeInappropriateLoggedOutErrorRoute"}'
'</script></body></html>',
200,
);

await expectLater(
const FacebookExtractor().extract(
'https://www.facebook.com/reel/1831368848220203/',
),
throwsA(
isA<ExtractionException>().having(
(error) => error.failure.kind,
'failure kind',
ExtractionFailureKind.facebookAgeRestricted,
),
),
);
});

test(
'Facebook reads an Open Graph video from a share landing page',
() async {
Expand Down