[tvn24] support nextjs frontend (playwright required) and magazine
This commit is contained in:
parent
1804387116
commit
339db8d1d2
|
@ -1,25 +1,28 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
|
extract_attributes,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
NO_DEFAULT,
|
NO_DEFAULT,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
)
|
)
|
||||||
|
from ..playwright import PlaywrightHelper
|
||||||
|
|
||||||
|
|
||||||
class TVN24IE(InfoExtractor):
|
class TVN24IE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)'
|
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl)/(?:[^/]+/)*[^/?#\s]+[,-](?P<id>\d+)(?:\.html)?'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
|
'url': 'https://tvn24.pl/polska/edyta-gorniak-napisala-o-statystach-w-szpitalach-udajacych-chorych-na-covid-19-jerzy-polaczek-i-marek-posobkiewicz-odpowiadaja-zapraszamy-4747899',
|
||||||
'md5': 'fbdec753d7bc29d96036808275f2130c',
|
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '1584444',
|
'id': '4744453',
|
||||||
'ext': 'mp4',
|
'ext': 'm3u8',
|
||||||
'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
|
'title': '"Nie wszyscy \'statyści\' wychodzą na własnych nogach". Chorzy na COVID-19 odpowiadają Edycie Górniak',
|
||||||
'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości Szkła kontaktowego.',
|
'description': 'md5:aa98c393df243a69203cfb6769b8db51',
|
||||||
'thumbnail': 're:https?://.*[.]jpeg',
|
|
||||||
}
|
}
|
||||||
}, {
|
}, {
|
||||||
# different layout
|
# different layout
|
||||||
|
@ -33,6 +36,14 @@ class TVN24IE(InfoExtractor):
|
||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://tvn24.pl/magazyn-tvn24/zrobimy-z-ciebie-mezczyzne,242,4189',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '4189',
|
||||||
|
'title': 'Zrobimy z ciebie mężczyznę',
|
||||||
|
'description': 'Milo nie miało myśli erotycznych. Tak boleśnie myślało o własnym ciele, że nie potrafiło myśleć o nim w towarzystwie innych ciał. Ale żeby zmienić ciało, m...',
|
||||||
|
},
|
||||||
|
'playlist_count': 2,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
|
'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
@ -46,10 +57,20 @@ class TVN24IE(InfoExtractor):
|
||||||
'url': 'https://www.tvn24.pl/magazyn-tvn24/angie-w-jednej-czwartej-polka-od-szarej-myszki-do-cesarzowej-europy,119,2158',
|
'url': 'https://www.tvn24.pl/magazyn-tvn24/angie-w-jednej-czwartej-polka-od-szarej-myszki-do-cesarzowej-europy,119,2158',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
_REQUIRES_PLAYWRIGHT = True
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
domain, display_id = mobj.group('domain', 'id')
|
||||||
|
|
||||||
|
if '/magazyn-tvn24/' in url:
|
||||||
|
return self._handle_magazine_frontend(url, display_id)
|
||||||
|
elif domain in ('tvn24.pl', 'tvn24bis.pl'):
|
||||||
|
return self._handle_nextjs_frontend(url, display_id)
|
||||||
|
else:
|
||||||
|
return self._handle_old_frontend(url, display_id)
|
||||||
|
|
||||||
|
def _handle_old_frontend(self, url, display_id):
|
||||||
webpage = self._download_webpage(url, display_id)
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
title = self._og_search_title(
|
title = self._og_search_title(
|
||||||
|
@ -101,3 +122,71 @@ class TVN24IE(InfoExtractor):
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _handle_magazine_frontend(self, url, display_id):
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
entries = []
|
||||||
|
for vid_el in re.finditer(r'(?P<video><div\b[^>]+\bdata-src=[^>]+>)\s*(?:</[^>]+>\s*)*<figcaption>(?P<title>(?:.|\s)+?)</figcaption>', webpage):
|
||||||
|
vid = extract_attributes(vid_el.group('video'))
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for fmt_name, fmt_url in self._parse_json(unescapeHTML(vid['data-quality']), display_id).items():
|
||||||
|
formats.append({
|
||||||
|
'format_id': fmt_name,
|
||||||
|
'height': int_or_none(fmt_name[:-1]),
|
||||||
|
'url': fmt_url,
|
||||||
|
})
|
||||||
|
|
||||||
|
self._sort_formats(formats)
|
||||||
|
entries.append({
|
||||||
|
'id': vid['data-video-id'],
|
||||||
|
'title': clean_html(vid_el.group('title')),
|
||||||
|
'formats': formats,
|
||||||
|
'thumbnail': vid.get('data-poster'),
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'playlist',
|
||||||
|
'id': display_id,
|
||||||
|
'entries': entries,
|
||||||
|
'title': self._og_search_title(webpage),
|
||||||
|
'description': self._og_search_description(webpage),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _handle_nextjs_frontend(self, url, display_id):
|
||||||
|
pwh = PlaywrightHelper(self)
|
||||||
|
page = pwh.open_page(url, display_id)
|
||||||
|
page.route(re.compile(r'(\.(png|jpg|svg|css)$)'), lambda route: route.abort())
|
||||||
|
|
||||||
|
# GDPR consent, required to play video
|
||||||
|
page.wait_for_selector('#onetrust-accept-btn-handler')
|
||||||
|
page.click('#onetrust-accept-btn-handler')
|
||||||
|
|
||||||
|
with page.expect_request(
|
||||||
|
lambda r: re.match(r'https?://(?:www\.)?tvn24(?:bis)?\.pl/api/[A-Za-z\d+-]+/plst', r.url),
|
||||||
|
timeout=20000) as plst_req:
|
||||||
|
# tip: always collect the request data before closing browser
|
||||||
|
plst_url = plst_req.value.url
|
||||||
|
title = page.eval_on_selector('meta[property="og:title"]', 'el => el.content')
|
||||||
|
description = page.eval_on_selector('meta[property="og:description"]', 'el => el.content')
|
||||||
|
pwh.browser_stop()
|
||||||
|
data = self._download_json(plst_url, display_id)
|
||||||
|
|
||||||
|
movie = data['movie']
|
||||||
|
sources = movie['video']['sources']
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
if sources.get('hls'):
|
||||||
|
formats.extend(self._extract_m3u8_formats(sources['hls']['url'], display_id))
|
||||||
|
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': movie['info']['id'],
|
||||||
|
'formats': formats,
|
||||||
|
'title': title,
|
||||||
|
'description': description,
|
||||||
|
'duration': movie['info']['total_time'],
|
||||||
|
'is_live': movie['video']['is_live'],
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue