[pornhub] Improve extraction and extract all formats (closes #12166, closes #15891, closes #16262, closes #16959)

This commit is contained in:
Sergey M․ 2018-07-14 18:05:06 +07:00
parent 40a051fa9f
commit 79367a9820
No known key found for this signature in database
GPG key ID: 2C393E0F18A9236D

View file

@ -4,28 +4,21 @@ from __future__ import unicode_literals
import functools import functools
import itertools import itertools
import operator import operator
# import os
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_HTTPError, compat_HTTPError,
# compat_urllib_parse_unquote, compat_str,
# compat_urllib_parse_unquote_plus,
# compat_urllib_parse_urlparse,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
int_or_none, int_or_none,
js_to_json, js_to_json,
orderedSet, orderedSet,
# sanitized_Request,
remove_quotes, remove_quotes,
str_to_int, str_to_int,
) )
# from ..aes import (
# aes_decrypt_text
# )
class PornHubIE(InfoExtractor): class PornHubIE(InfoExtractor):
@ -62,7 +55,7 @@ class PornHubIE(InfoExtractor):
'id': '1331683002', 'id': '1331683002',
'ext': 'mp4', 'ext': 'mp4',
'title': '重庆婷婷女王足交', 'title': '重庆婷婷女王足交',
'uploader': 'cj397186295', 'uploader': 'Unknown',
'duration': 1753, 'duration': 1753,
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
@ -121,7 +114,7 @@ class PornHubIE(InfoExtractor):
self._set_cookie('pornhub.com', 'platform', platform) self._set_cookie('pornhub.com', 'platform', platform)
return self._download_webpage( return self._download_webpage(
'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id, 'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id,
video_id) video_id, 'Downloading %s webpage' % platform)
webpage = dl_webpage('pc') webpage = dl_webpage('pc')
@ -134,6 +127,43 @@ class PornHubIE(InfoExtractor):
'PornHub said: %s' % error_msg, 'PornHub said: %s' % error_msg,
expected=True, video_id=video_id) expected=True, video_id=video_id)
# video_title from flashvars contains whitespace instead of non-ASCII (see
# http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
# on that anymore.
title = self._html_search_meta(
'twitter:title', webpage, default=None) or self._search_regex(
(r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'),
webpage, 'title', group='title')
video_urls = []
video_urls_set = set()
flashvars = self._parse_json(
self._search_regex(
r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
video_id)
if flashvars:
thumbnail = flashvars.get('image_url')
duration = int_or_none(flashvars.get('video_duration'))
media_definitions = flashvars.get('mediaDefinitions')
if isinstance(media_definitions, list):
for definition in media_definitions:
if not isinstance(definition, dict):
continue
video_url = definition.get('videoUrl')
if not video_url or not isinstance(video_url, compat_str):
continue
if video_url in video_urls_set:
continue
video_urls_set.add(video_url)
video_urls.append(
(video_url, int_or_none(definition.get('quality'))))
else:
thumbnail, duration = [None] * 2
if not video_urls:
tv_webpage = dl_webpage('tv') tv_webpage = dl_webpage('tv')
assignments = self._search_regex( assignments = self._search_regex(
@ -162,29 +192,33 @@ class PornHubIE(InfoExtractor):
js_vars[vname] = parse_js_value(value) js_vars[vname] = parse_js_value(value)
video_url = js_vars['mediastring'] video_url = js_vars['mediastring']
if video_url not in video_urls_set:
video_urls.append((video_url, None))
video_urls_set.add(video_url)
title = self._search_regex( for mobj in re.finditer(
r'<h1>([^>]+)</h1>', tv_webpage, 'title', default=None) r'<a[^>]+\bclass=["\']downloadBtn\b[^>]+\bhref=(["\'])(?P<url>(?:(?!\1).)+)\1',
webpage):
video_url = mobj.group('url')
if video_url not in video_urls_set:
video_urls.append((video_url, None))
video_urls_set.add(video_url)
# video_title from flashvars contains whitespace instead of non-ASCII (see formats = []
# http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying for video_url, height in video_urls:
# on that anymore. tbr = None
title = title or self._html_search_meta( mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
'twitter:title', webpage, default=None) or self._search_regex( if mobj:
(r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)', if not height:
r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1', height = int(mobj.group('height'))
r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'), tbr = int(mobj.group('tbr'))
webpage, 'title', group='title') formats.append({
'url': video_url,
flashvars = self._parse_json( 'format_id': '%dp' % height if height else None,
self._search_regex( 'height': height,
r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'), 'tbr': tbr,
video_id) })
if flashvars: self._sort_formats(formats)
thumbnail = flashvars.get('image_url')
duration = int_or_none(flashvars.get('video_duration'))
else:
title, thumbnail, duration = [None] * 3
video_uploader = self._html_search_regex( video_uploader = self._html_search_regex(
r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:user|channel)s/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<', r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:user|channel)s/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
@ -210,7 +244,6 @@ class PornHubIE(InfoExtractor):
return { return {
'id': video_id, 'id': video_id,
'url': video_url,
'uploader': video_uploader, 'uploader': video_uploader,
'title': title, 'title': title,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
@ -219,7 +252,7 @@ class PornHubIE(InfoExtractor):
'like_count': like_count, 'like_count': like_count,
'dislike_count': dislike_count, 'dislike_count': dislike_count,
'comment_count': comment_count, 'comment_count': comment_count,
# 'formats': formats, 'formats': formats,
'age_limit': 18, 'age_limit': 18,
'tags': tags, 'tags': tags,
'categories': categories, 'categories': categories,