From 339db8d1d20c6323b7d42b95e234fe8874a09010 Mon Sep 17 00:00:00 2001
From: Laura Liberda <laura@selfisekai.rocks>
Date: Fri, 22 Jan 2021 19:59:00 +0100
Subject: [PATCH] [tvn24] support nextjs frontend (playwright required) and
 magazine

---
 haruhi_dl/extractor/tvn24.py | 107 ++++++++++++++++++++++++++++++++---
 1 file changed, 98 insertions(+), 9 deletions(-)

diff --git a/haruhi_dl/extractor/tvn24.py b/haruhi_dl/extractor/tvn24.py
index de0fb5063..10fe60c7b 100644
--- a/haruhi_dl/extractor/tvn24.py
+++ b/haruhi_dl/extractor/tvn24.py
@@ -1,25 +1,28 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    extract_attributes,
     int_or_none,
     NO_DEFAULT,
     unescapeHTML,
 )
+from ..playwright import PlaywrightHelper
 
 
 class TVN24IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl)/(?:[^/]+/)*[^/?#\s]+[,-](?P<id>\d+)(?:\.html)?'
     _TESTS = [{
-        'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
-        'md5': 'fbdec753d7bc29d96036808275f2130c',
+        'url': 'https://tvn24.pl/polska/edyta-gorniak-napisala-o-statystach-w-szpitalach-udajacych-chorych-na-covid-19-jerzy-polaczek-i-marek-posobkiewicz-odpowiadaja-zapraszamy-4747899',
         'info_dict': {
-            'id': '1584444',
-            'ext': 'mp4',
-            'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
-            'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości Szkła kontaktowego.',
-            'thumbnail': 're:https?://.*[.]jpeg',
+            'id': '4744453',
+            'ext': 'm3u8',
+            'title': '"Nie wszyscy \'statyści\' wychodzą na własnych nogach". Chorzy na COVID-19 odpowiadają Edycie Górniak',
+            'description': 'md5:aa98c393df243a69203cfb6769b8db51',
         }
     }, {
         # different layout
@@ -33,6 +36,14 @@ class TVN24IE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://tvn24.pl/magazyn-tvn24/zrobimy-z-ciebie-mezczyzne,242,4189',
+        'info_dict': {
+            'id': '4189',
+            'title': 'Zrobimy z ciebie mężczyznę',
+            'description': 'Milo nie miało myśli erotycznych. Tak boleśnie myślało o własnym ciele, że nie potrafiło myśleć o nim w towarzystwie innych ciał. Ale żeby zmienić ciało, m...',
+        },
+        'playlist_count': 2,
     }, {
         'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
         'only_matching': True,
@@ -46,10 +57,20 @@ class TVN24IE(InfoExtractor):
         'url': 'https://www.tvn24.pl/magazyn-tvn24/angie-w-jednej-czwartej-polka-od-szarej-myszki-do-cesarzowej-europy,119,2158',
         'only_matching': True,
     }]
+    _REQUIRES_PLAYWRIGHT = True
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        domain, display_id = mobj.group('domain', 'id')
 
+        if '/magazyn-tvn24/' in url:
+            return self._handle_magazine_frontend(url, display_id)
+        elif domain in ('tvn24.pl', 'tvn24bis.pl'):
+            return self._handle_nextjs_frontend(url, display_id)
+        else:
+            return self._handle_old_frontend(url, display_id)
+
+    def _handle_old_frontend(self, url, display_id):
         webpage = self._download_webpage(url, display_id)
 
         title = self._og_search_title(
@@ -101,3 +122,71 @@ class TVN24IE(InfoExtractor):
             'thumbnail': thumbnail,
             'formats': formats,
         }
+
+    def _handle_magazine_frontend(self, url, display_id):
+        webpage = self._download_webpage(url, display_id)
+
+        entries = []
+        for vid_el in re.finditer(r'(?P<video><div\b[^>]+\bdata-src=[^>]+>)\s*(?:</[^>]+>\s*)*<figcaption>(?P<title>(?:.|\s)+?)</figcaption>', webpage):
+            vid = extract_attributes(vid_el.group('video'))
+
+            formats = []
+            for fmt_name, fmt_url in self._parse_json(unescapeHTML(vid['data-quality']), display_id).items():
+                formats.append({
+                    'format_id': fmt_name,
+                    'height': int_or_none(fmt_name[:-1]),
+                    'url': fmt_url,
+                })
+
+            self._sort_formats(formats)
+            entries.append({
+                'id': vid['data-video-id'],
+                'title': clean_html(vid_el.group('title')),
+                'formats': formats,
+                'thumbnail': vid.get('data-poster'),
+            })
+
+        return {
+            '_type': 'playlist',
+            'id': display_id,
+            'entries': entries,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+        }
+
+    def _handle_nextjs_frontend(self, url, display_id):
+        pwh = PlaywrightHelper(self)
+        page = pwh.open_page(url, display_id)
+        page.route(re.compile(r'(\.(png|jpg|svg|css)$)'), lambda route: route.abort())
+
+        # GDPR consent, required to play video
+        page.wait_for_selector('#onetrust-accept-btn-handler')
+        page.click('#onetrust-accept-btn-handler')
+
+        with page.expect_request(
+                lambda r: re.match(r'https?://(?:www\.)?tvn24(?:bis)?\.pl/api/[A-Za-z\d+-]+/plst', r.url),
+                timeout=20000) as plst_req:
+            # tip: always collect the request data before closing browser
+            plst_url = plst_req.value.url
+            title = page.eval_on_selector('meta[property="og:title"]', 'el => el.content')
+            description = page.eval_on_selector('meta[property="og:description"]', 'el => el.content')
+            pwh.browser_stop()
+            data = self._download_json(plst_url, display_id)
+
+            movie = data['movie']
+            sources = movie['video']['sources']
+
+            formats = []
+            if sources.get('hls'):
+                formats.extend(self._extract_m3u8_formats(sources['hls']['url'], display_id))
+
+            self._sort_formats(formats)
+
+            return {
+                'id': movie['info']['id'],
+                'formats': formats,
+                'title': title,
+                'description': description,
+                'duration': movie['info']['total_time'],
+                'is_live': movie['video']['is_live'],
+            }