laudom/haruhi-dl
2
0
Fork 0
Code Issues 13 Pull requests Packages Projects Releases Wiki Activity

[pornhub] Implement lazy playlist extraction

Browse source
This commit is contained in:
=?UTF-8?q?Sergey=20M=E2=80=A4?= 2021-02-26 18:02:37 +01:00 committed by Dominika
parent efbbc4fb90
commit b37f3b7703
1 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
haruhi_dl/extractor/pornhub.py
View file

@ -545,13 +545,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
<button[^>]+\bid=["\']moreDataBtn
''', webpage) is not None
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
host = mobj.group('host')
item_id = mobj.group('id')
self._login(host)
def _entries(self, url, host, item_id):
page = self._extract_page(url)
VIDEOS = '/videos'
@ -564,7 +558,6 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
def is_404(e):
return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
entries = []
base_url = url
has_page = page is not None
first_page = page if has_page else 1
@ -588,11 +581,19 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
page_entries = self._extract_entries(webpage, host)
if not page_entries:
break
entries.extend(page_entries)
for e in page_entries:
yield e
if not self._has_more(webpage):
break
return self.playlist_result(orderedSet(entries), item_id)
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
host = mobj.group('host')
item_id = mobj.group('id')
self._login(host)
return self.playlist_result(self._entries(url, host, item_id), item_id)
class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):