Wed Nov 18 17:35:15 2020 UTC ()
youtube-dl: Update to 20201118

pkgsrc changes:
 - Remove patch-youtube__dl_extractor_bandcamp.py, fixed differently upstream
 - Update patch-youtube__dl_extractor_rai.py to current rai extractor
 - Add a reference to upstream pull request in patch-youtube__dl_extractor_la7.py

Changes:
2020.11.18
----------
Extractors
* [spiegel] Fix extraction (#24206, #24767)
* [youtube] Improve extraction
    + Add support for --no-playlist (#27009)
    * Improve playlist and mix extraction (#26390, #26509, #26534, #27011)
    + Extract playlist uploader data
* [youtube:tab] Fix view count extraction (#27051)
* [malltv] Fix extraction (#27035)
+ [bandcamp] Extract playlist description (#22684)
* [urplay] Fix extraction (#26828)
* [youtube:tab] Fix playlist title extraction (#27015)
* [youtube] Fix chapters extraction (#26005)

2020.11.17
----------
Core
* [utils] Skip ! prefixed code in js_to_json

Extractors
* [youtube:tab] Fix extraction with cookies provided (#27005)
* [lrt] Fix extraction with empty tags (#20264)
+ [ndr:embed:base] Extract subtitles (#25447, #26106)
+ [servus] Add support for pm-wissen.com (#25869)
* [servus] Fix extraction (#26872, #26967, #26983, #27000)
* [xtube] Fix extraction (#26996)
* [lrt] Fix extraction
+ [lbry] Add support for lbry.tv
+ [condenast] Extract subtitles
* [condenast] Fix extraction
* [bandcamp] Fix extraction (#26681, #26684)
* [rai] Fix RaiPlay extraction (#26064, #26096)
* [vlive] Fix extraction
* [usanetwork] Fix extraction
* [nbc] Fix NBCNews/Today/MSNBC extraction
* [cnbc] Fix extraction


(leot)
diff -r1.218 -r1.219 pkgsrc/net/youtube-dl/Makefile
diff -r1.99 -r1.100 pkgsrc/net/youtube-dl/PLIST
diff -r1.200 -r1.201 pkgsrc/net/youtube-dl/distinfo
diff -r1.1 -r0 pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_bandcamp.py
diff -r1.2 -r1.3 pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_la7.py
diff -r1.2 -r1.3 pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_rai.py

cvs diff -r1.218 -r1.219 pkgsrc/net/youtube-dl/Makefile (expand / switch to unified diff)

--- pkgsrc/net/youtube-dl/Makefile 2020/11/12 14:41:38 1.218
+++ pkgsrc/net/youtube-dl/Makefile 2020/11/18 17:35:15 1.219
@@ -1,18 +1,18 @@ @@ -1,18 +1,18 @@
1# $NetBSD: Makefile,v 1.218 2020/11/12 14:41:38 leot Exp $ 1# $NetBSD: Makefile,v 1.219 2020/11/18 17:35:15 leot Exp $
2 2
3# XXX: VERSION_DATE can contains also an optional part that indicates 3# XXX: VERSION_DATE can contains also an optional part that indicates
4# XXX: possible same day revisions. PKGNAME preserves that dotted part as is. 4# XXX: possible same day revisions. PKGNAME preserves that dotted part as is.
5VERSION_DATE= 2020.11.12 5VERSION_DATE= 2020.11.18
6DISTNAME= youtube-dl-${VERSION_DATE} 6DISTNAME= youtube-dl-${VERSION_DATE}
7PKGNAME= ${DISTNAME:S/.//:S/.//} 7PKGNAME= ${DISTNAME:S/.//:S/.//}
8CATEGORIES= net 8CATEGORIES= net
9MASTER_SITES= https://youtube-dl.org/downloads/${VERSION_DATE}/ 9MASTER_SITES= https://youtube-dl.org/downloads/${VERSION_DATE}/
10 10
11MAINTAINER= leot@NetBSD.org 11MAINTAINER= leot@NetBSD.org
12HOMEPAGE= https://ytdl-org.github.io/youtube-dl/ 12HOMEPAGE= https://ytdl-org.github.io/youtube-dl/
13COMMENT= Download videos from youtube.com 13COMMENT= Download videos from youtube.com
14LICENSE= public-domain 14LICENSE= public-domain
15 15
16USE_LANGUAGES= # none 16USE_LANGUAGES= # none
17EGG_NAME= youtube_dl-${VERSION_DATE:S/.0/./g} 17EGG_NAME= youtube_dl-${VERSION_DATE:S/.0/./g}
18WRKSRC= ${WRKDIR}/youtube-dl 18WRKSRC= ${WRKDIR}/youtube-dl

cvs diff -r1.99 -r1.100 pkgsrc/net/youtube-dl/PLIST (expand / switch to unified diff)

--- pkgsrc/net/youtube-dl/PLIST 2020/02/16 19:28:47 1.99
+++ pkgsrc/net/youtube-dl/PLIST 2020/11/18 17:35:15 1.100
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1@comment $NetBSD: PLIST,v 1.99 2020/02/16 19:28:47 leot Exp $ 1@comment $NetBSD: PLIST,v 1.100 2020/11/18 17:35:15 leot Exp $
2bin/youtube-dl 2bin/youtube-dl
3${PYSITELIB}/${EGG_INFODIR}/PKG-INFO 3${PYSITELIB}/${EGG_INFODIR}/PKG-INFO
4${PYSITELIB}/${EGG_INFODIR}/SOURCES.txt 4${PYSITELIB}/${EGG_INFODIR}/SOURCES.txt
5${PYSITELIB}/${EGG_INFODIR}/dependency_links.txt 5${PYSITELIB}/${EGG_INFODIR}/dependency_links.txt
6${PYSITELIB}/${EGG_INFODIR}/entry_points.txt 6${PYSITELIB}/${EGG_INFODIR}/entry_points.txt
7${PYSITELIB}/${EGG_INFODIR}/top_level.txt 7${PYSITELIB}/${EGG_INFODIR}/top_level.txt
8${PYSITELIB}/youtube_dl/YoutubeDL.py 8${PYSITELIB}/youtube_dl/YoutubeDL.py
9${PYSITELIB}/youtube_dl/YoutubeDL.pyc 9${PYSITELIB}/youtube_dl/YoutubeDL.pyc
10${PYSITELIB}/youtube_dl/YoutubeDL.pyo 10${PYSITELIB}/youtube_dl/YoutubeDL.pyo
11${PYSITELIB}/youtube_dl/__init__.py 11${PYSITELIB}/youtube_dl/__init__.py
12${PYSITELIB}/youtube_dl/__init__.pyc 12${PYSITELIB}/youtube_dl/__init__.pyc
13${PYSITELIB}/youtube_dl/__init__.pyo 13${PYSITELIB}/youtube_dl/__init__.pyo
14${PYSITELIB}/youtube_dl/__main__.py 14${PYSITELIB}/youtube_dl/__main__.py
@@ -958,26 +958,29 @@ ${PYSITELIB}/youtube_dl/extractor/ku6.py @@ -958,26 +958,29 @@ ${PYSITELIB}/youtube_dl/extractor/ku6.py
958${PYSITELIB}/youtube_dl/extractor/ku6.pyo 958${PYSITELIB}/youtube_dl/extractor/ku6.pyo
959${PYSITELIB}/youtube_dl/extractor/kusi.py 959${PYSITELIB}/youtube_dl/extractor/kusi.py
960${PYSITELIB}/youtube_dl/extractor/kusi.pyc 960${PYSITELIB}/youtube_dl/extractor/kusi.pyc
961${PYSITELIB}/youtube_dl/extractor/kusi.pyo 961${PYSITELIB}/youtube_dl/extractor/kusi.pyo
962${PYSITELIB}/youtube_dl/extractor/kuwo.py 962${PYSITELIB}/youtube_dl/extractor/kuwo.py
963${PYSITELIB}/youtube_dl/extractor/kuwo.pyc 963${PYSITELIB}/youtube_dl/extractor/kuwo.pyc
964${PYSITELIB}/youtube_dl/extractor/kuwo.pyo 964${PYSITELIB}/youtube_dl/extractor/kuwo.pyo
965${PYSITELIB}/youtube_dl/extractor/la7.py 965${PYSITELIB}/youtube_dl/extractor/la7.py
966${PYSITELIB}/youtube_dl/extractor/la7.pyc 966${PYSITELIB}/youtube_dl/extractor/la7.pyc
967${PYSITELIB}/youtube_dl/extractor/la7.pyo 967${PYSITELIB}/youtube_dl/extractor/la7.pyo
968${PYSITELIB}/youtube_dl/extractor/laola1tv.py 968${PYSITELIB}/youtube_dl/extractor/laola1tv.py
969${PYSITELIB}/youtube_dl/extractor/laola1tv.pyc 969${PYSITELIB}/youtube_dl/extractor/laola1tv.pyc
970${PYSITELIB}/youtube_dl/extractor/laola1tv.pyo 970${PYSITELIB}/youtube_dl/extractor/laola1tv.pyo
 971${PYSITELIB}/youtube_dl/extractor/lbry.py
 972${PYSITELIB}/youtube_dl/extractor/lbry.pyc
 973${PYSITELIB}/youtube_dl/extractor/lbry.pyo
971${PYSITELIB}/youtube_dl/extractor/lci.py 974${PYSITELIB}/youtube_dl/extractor/lci.py
972${PYSITELIB}/youtube_dl/extractor/lci.pyc 975${PYSITELIB}/youtube_dl/extractor/lci.pyc
973${PYSITELIB}/youtube_dl/extractor/lci.pyo 976${PYSITELIB}/youtube_dl/extractor/lci.pyo
974${PYSITELIB}/youtube_dl/extractor/lcp.py 977${PYSITELIB}/youtube_dl/extractor/lcp.py
975${PYSITELIB}/youtube_dl/extractor/lcp.pyc 978${PYSITELIB}/youtube_dl/extractor/lcp.pyc
976${PYSITELIB}/youtube_dl/extractor/lcp.pyo 979${PYSITELIB}/youtube_dl/extractor/lcp.pyo
977${PYSITELIB}/youtube_dl/extractor/lecture2go.py 980${PYSITELIB}/youtube_dl/extractor/lecture2go.py
978${PYSITELIB}/youtube_dl/extractor/lecture2go.pyc 981${PYSITELIB}/youtube_dl/extractor/lecture2go.pyc
979${PYSITELIB}/youtube_dl/extractor/lecture2go.pyo 982${PYSITELIB}/youtube_dl/extractor/lecture2go.pyo
980${PYSITELIB}/youtube_dl/extractor/lecturio.py 983${PYSITELIB}/youtube_dl/extractor/lecturio.py
981${PYSITELIB}/youtube_dl/extractor/lecturio.pyc 984${PYSITELIB}/youtube_dl/extractor/lecturio.pyc
982${PYSITELIB}/youtube_dl/extractor/lecturio.pyo 985${PYSITELIB}/youtube_dl/extractor/lecturio.pyo
983${PYSITELIB}/youtube_dl/extractor/leeco.py 986${PYSITELIB}/youtube_dl/extractor/leeco.py
@@ -1699,29 +1702,26 @@ ${PYSITELIB}/youtube_dl/extractor/soundg @@ -1699,29 +1702,26 @@ ${PYSITELIB}/youtube_dl/extractor/soundg
1699${PYSITELIB}/youtube_dl/extractor/soundgasm.pyo 1702${PYSITELIB}/youtube_dl/extractor/soundgasm.pyo
1700${PYSITELIB}/youtube_dl/extractor/southpark.py 1703${PYSITELIB}/youtube_dl/extractor/southpark.py
1701${PYSITELIB}/youtube_dl/extractor/southpark.pyc 1704${PYSITELIB}/youtube_dl/extractor/southpark.pyc
1702${PYSITELIB}/youtube_dl/extractor/southpark.pyo 1705${PYSITELIB}/youtube_dl/extractor/southpark.pyo
1703${PYSITELIB}/youtube_dl/extractor/spankbang.py 1706${PYSITELIB}/youtube_dl/extractor/spankbang.py
1704${PYSITELIB}/youtube_dl/extractor/spankbang.pyc 1707${PYSITELIB}/youtube_dl/extractor/spankbang.pyc
1705${PYSITELIB}/youtube_dl/extractor/spankbang.pyo 1708${PYSITELIB}/youtube_dl/extractor/spankbang.pyo
1706${PYSITELIB}/youtube_dl/extractor/spankwire.py 1709${PYSITELIB}/youtube_dl/extractor/spankwire.py
1707${PYSITELIB}/youtube_dl/extractor/spankwire.pyc 1710${PYSITELIB}/youtube_dl/extractor/spankwire.pyc
1708${PYSITELIB}/youtube_dl/extractor/spankwire.pyo 1711${PYSITELIB}/youtube_dl/extractor/spankwire.pyo
1709${PYSITELIB}/youtube_dl/extractor/spiegel.py 1712${PYSITELIB}/youtube_dl/extractor/spiegel.py
1710${PYSITELIB}/youtube_dl/extractor/spiegel.pyc 1713${PYSITELIB}/youtube_dl/extractor/spiegel.pyc
1711${PYSITELIB}/youtube_dl/extractor/spiegel.pyo 1714${PYSITELIB}/youtube_dl/extractor/spiegel.pyo
1712${PYSITELIB}/youtube_dl/extractor/spiegeltv.py 
1713${PYSITELIB}/youtube_dl/extractor/spiegeltv.pyc 
1714${PYSITELIB}/youtube_dl/extractor/spiegeltv.pyo 
1715${PYSITELIB}/youtube_dl/extractor/spike.py 1715${PYSITELIB}/youtube_dl/extractor/spike.py
1716${PYSITELIB}/youtube_dl/extractor/spike.pyc 1716${PYSITELIB}/youtube_dl/extractor/spike.pyc
1717${PYSITELIB}/youtube_dl/extractor/spike.pyo 1717${PYSITELIB}/youtube_dl/extractor/spike.pyo
1718${PYSITELIB}/youtube_dl/extractor/sport5.py 1718${PYSITELIB}/youtube_dl/extractor/sport5.py
1719${PYSITELIB}/youtube_dl/extractor/sport5.pyc 1719${PYSITELIB}/youtube_dl/extractor/sport5.pyc
1720${PYSITELIB}/youtube_dl/extractor/sport5.pyo 1720${PYSITELIB}/youtube_dl/extractor/sport5.pyo
1721${PYSITELIB}/youtube_dl/extractor/sportbox.py 1721${PYSITELIB}/youtube_dl/extractor/sportbox.py
1722${PYSITELIB}/youtube_dl/extractor/sportbox.pyc 1722${PYSITELIB}/youtube_dl/extractor/sportbox.pyc
1723${PYSITELIB}/youtube_dl/extractor/sportbox.pyo 1723${PYSITELIB}/youtube_dl/extractor/sportbox.pyo
1724${PYSITELIB}/youtube_dl/extractor/sportdeutschland.py 1724${PYSITELIB}/youtube_dl/extractor/sportdeutschland.py
1725${PYSITELIB}/youtube_dl/extractor/sportdeutschland.pyc 1725${PYSITELIB}/youtube_dl/extractor/sportdeutschland.pyc
1726${PYSITELIB}/youtube_dl/extractor/sportdeutschland.pyo 1726${PYSITELIB}/youtube_dl/extractor/sportdeutschland.pyo
1727${PYSITELIB}/youtube_dl/extractor/springboardplatform.py 1727${PYSITELIB}/youtube_dl/extractor/springboardplatform.py

cvs diff -r1.200 -r1.201 pkgsrc/net/youtube-dl/distinfo (expand / switch to unified diff)

--- pkgsrc/net/youtube-dl/distinfo 2020/11/12 14:41:38 1.200
+++ pkgsrc/net/youtube-dl/distinfo 2020/11/18 17:35:15 1.201
@@ -1,11 +1,10 @@ @@ -1,11 +1,10 @@
1$NetBSD: distinfo,v 1.200 2020/11/12 14:41:38 leot Exp $ 1$NetBSD: distinfo,v 1.201 2020/11/18 17:35:15 leot Exp $
2 2
3SHA1 (youtube-dl-2020.11.12.tar.gz) = 04e72d0b0a0e85b79a6c2ac93b7c85254b95b53b 3SHA1 (youtube-dl-2020.11.18.tar.gz) = e1b922ebc543f35ea7ee3de7e28e8deea7b97914
4RMD160 (youtube-dl-2020.11.12.tar.gz) = 2afd73b5c09463951086b29298489f0d203a2207 4RMD160 (youtube-dl-2020.11.18.tar.gz) = e526d2c4f297390cba92ae91d70223e9be3171e6
5SHA512 (youtube-dl-2020.11.12.tar.gz) = 7db373f6cc252635a3613ffe0b3b10640e262778105ebbd78b837fe019b0a2609032d2aeb81b239e000a86220aff99d2c018a9a6325adad6981a8ab64048131c 5SHA512 (youtube-dl-2020.11.18.tar.gz) = 110de857759b4c4bd0160242adebb3d8690bda2203a28a7b1a2ac1cdd9bca058702fd0b323010629e74bbb2df38f50c67b710bc2a6ad4cc907827ee013d0dbcf
6Size (youtube-dl-2020.11.12.tar.gz) = 3188015 bytes 6Size (youtube-dl-2020.11.18.tar.gz) = 3186065 bytes
7SHA1 (patch-setup.py) = a67074ae7cfe5e77847c2f610337ea553eddb69b 7SHA1 (patch-setup.py) = a67074ae7cfe5e77847c2f610337ea553eddb69b
8SHA1 (patch-youtube__dl_extractor_bandcamp.py) = 81855a3f4f8c03f61fe543eb339c0e67bf52682e 8SHA1 (patch-youtube__dl_extractor_la7.py) = 6c579f96e7ace1b64ef25fe8788b40bc4e7e67dd
9SHA1 (patch-youtube__dl_extractor_la7.py) = e246750808305343227060acdc5a38583ef071e9 9SHA1 (patch-youtube__dl_extractor_rai.py) = 5ec18da74c46f2195fe814d61ca044df4b70cc45
10SHA1 (patch-youtube__dl_extractor_rai.py) = 3dbad7852b38e7364a248a5c9851c50cd2ff9b38 
11SHA1 (patch-youtube__dl_postprocessor_ffmpeg.py) = f96676170a448d9205d542a7def4beca615a1490 10SHA1 (patch-youtube__dl_postprocessor_ffmpeg.py) = f96676170a448d9205d542a7def4beca615a1490

File Deleted: pkgsrc/net/youtube-dl/patches/Attic/patch-youtube__dl_extractor_bandcamp.py

cvs diff -r1.2 -r1.3 pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_la7.py (expand / switch to unified diff)

--- pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_la7.py 2020/03/23 20:32:23 1.2
+++ pkgsrc/net/youtube-dl/patches/patch-youtube__dl_extractor_la7.py 2020/11/18 17:35:15 1.3
@@ -1,62 +1,67 @@ @@ -1,62 +1,67 @@
1$NetBSD: patch-youtube__dl_extractor_la7.py,v 1.2 2020/03/23 20:32:23 leot Exp $ 1$NetBSD: patch-youtube__dl_extractor_la7.py,v 1.3 2020/11/18 17:35:15 leot Exp $
2 2
3[la7] Fix extraction (closes #23323) 3[la7] Fix extraction (closes #23323)
4 4
5Parsing `videoParams' or `videoLa7' JavaScript snippets as JSON is getting too 5Parsing `videoParams' or `videoLa7' JavaScript snippets as JSON is getting too
6hard for js_to_json. Just extract the `vid' from there and use _search_og_* 6hard for js_to_json. Just extract the `vid' from there and use _search_og_*
7for all other data. 7for all other data.
8 8
9Remove the 2nd test: unfortunately the page are removed after a couple of 9Remove the 2nd test: unfortunately the page are removed after a couple of
10days/weeks. 10days/weeks.
11 11
12Thanks to Elia Geretto for spotting and suggesting several problems with the 12Thanks to Elia Geretto for spotting and suggesting several problems with the
13tests! 13tests!
14 14
15Closes #23323. 15Closes #23323.
16 16
17--- youtube_dl/extractor/la7.py.orig 17Shared upstream via:
 18
 19 https://github.com/ytdl-org/youtube-dl/pull/23641
 20
 21--- youtube_dl/extractor/la7.py.orig 2020-11-16 21:04:10.000000000 +0000
18+++ youtube_dl/extractor/la7.py 22+++ youtube_dl/extractor/la7.py
19@@ -3,7 +3,6 @@ from __future__ import unicode_literals 23@@ -3,7 +3,6 @@ from __future__ import unicode_literals
20  24
21 from .common import InfoExtractor 25 from .common import InfoExtractor
22 from ..utils import ( 26 from ..utils import (
23- js_to_json, 27- js_to_json,
24 smuggle_url, 28 smuggle_url,
25 ) 29 )
26  30
27@@ -23,21 +22,12 @@ class LA7IE(InfoExtractor): 31@@ -23,22 +22,13 @@ class LA7IE(InfoExtractor):
28 'id': '0_42j6wd36', 32 'id': '0_42j6wd36',
29 'ext': 'mp4', 33 'ext': 'mp4',
30 'title': 'Inc.Cool8', 34 'title': 'Inc.Cool8',
31- 'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico', 35- 'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
32+ 'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico', 36+ 'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
33 'thumbnail': 're:^https?://.*', 37 'thumbnail': 're:^https?://.*',
34 'uploader_id': 'kdla7pillole@iltrovatore.it', 38 'uploader_id': 'kdla7pillole@iltrovatore.it',
35 'timestamp': 1443814869, 39 'timestamp': 1443814869,
36 'upload_date': '20151002', 40 'upload_date': '20151002',
37 }, 41 },
38- }, { 42 }, {
39- # 'src' is a dictionary 43- # 'src' is a dictionary
40- 'url': 'http://tg.la7.it/repliche-tgla7?id=189080', 44- 'url': 'http://tg.la7.it/repliche-tgla7?id=189080',
41- 'md5': '6b0d8888d286e39870208dfeceaf456b', 45- 'md5': '6b0d8888d286e39870208dfeceaf456b',
42- 'info_dict': { 46- 'info_dict': {
43- 'id': '189080', 47- 'id': '189080',
44- 'ext': 'mp4', 48- 'ext': 'mp4',
45- 'title': 'TG LA7', 49- 'title': 'TG LA7',
46- }, 50- },
47 }, { 51- }, {
48 'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077', 52 'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
49 'only_matching': True, 53 'only_matching': True,
 54 }]
50@@ -48,20 +38,19 @@ class LA7IE(InfoExtractor): 55@@ -48,20 +38,19 @@ class LA7IE(InfoExtractor):
51  56
52 webpage = self._download_webpage(url, video_id) 57 webpage = self._download_webpage(url, video_id)
53  58
54- player_data = self._parse_json( 59- player_data = self._parse_json(
55- self._search_regex( 60- self._search_regex(
56- [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'], 61- [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
57- webpage, 'player data'), 62- webpage, 'player data'),
58- video_id, transform_source=js_to_json) 63- video_id, transform_source=js_to_json)
59+ player_data = self._search_regex( 64+ player_data = self._search_regex(
60+ [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'], 65+ [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
61+ webpage, 'player data') 66+ webpage, 'player data')
62+ vid = self._search_regex(r'vid\s*:\s*"(.+?)",', player_data, 'vid') 67+ vid = self._search_regex(r'vid\s*:\s*"(.+?)",', player_data, 'vid')

cvs diff -r1.2 -r1.3 pkgsrc/net/youtube-dl/patches/Attic/patch-youtube__dl_extractor_rai.py (expand / switch to unified diff)

--- pkgsrc/net/youtube-dl/patches/Attic/patch-youtube__dl_extractor_rai.py 2020/09/06 09:54:34 1.2
+++ pkgsrc/net/youtube-dl/patches/Attic/patch-youtube__dl_extractor_rai.py 2020/11/18 17:35:15 1.3
@@ -1,194 +1,157 @@ @@ -1,194 +1,157 @@
1$NetBSD: patch-youtube__dl_extractor_rai.py,v 1.2 2020/09/06 09:54:34 leot Exp $ 1$NetBSD: patch-youtube__dl_extractor_rai.py,v 1.3 2020/11/18 17:35:15 leot Exp $
2 2
3[rai] Fix extraction for recent raiplay.it updates 3[rai] Fix extraction for recent raiplay.it updates
4 4
5- Introduce _BASE_URL in RaiBaseIE class so it could be reused as base for the 5- Introduce _BASE_URL in RaiBaseIE class so it could be reused as base for the
6 several subextractors. 6 several subextractors.
7- Remove first test of RaiPlayIE, it is no longer available 7- Remove first test of RaiPlayIE: it is no longer available
8- Adjust RaiPlayIE to recent raiplay.it updates, make it extension agnostic 8- Make RaiPlayIE extension-agnostic (passing possible `.json' URLs is now
9 (passing possible `.json' URLs is now supported too) and update test 9 supported too)
10 info_dict. 
11- Adjust RaiPlayLiveIE to recent raiplay.it updates. Passing it as 10- Adjust RaiPlayLiveIE to recent raiplay.it updates. Passing it as
12 `url_transparent' is no longer supported (there is no longer an accessible 11 `url_transparent' is no longer supported (there is no longer an accessible
13 ContentItem) 12 ContentItem)
14- Adjust RaiPlayPlaylistIE to recent raiplay.it updates and instruct it about 13- Adjust RaiPlayPlaylistIE to recent raiplay.it updates and instruct it about
15 ContentSet-s. 14 ContentSet-s.
16- Update a RaiIE test and remove two tests that are no longer availables 15- Update a RaiIE test and remove two tests that are no longer availables
17 16
18This fix issue #22923, #22906 and supersedes #23006. 17This fix issue #22923, #22906 and supersedes #23006 and #23040.
19 18
20Shared upstream via: 19Shared upstream via:
21 20
22 https://github.com/ytdl-org/youtube-dl/pull/23040 21 https://github.com/ytdl-org/youtube-dl/pull/27077
23 22
24--- youtube_dl/extractor/rai.py.orig 23--- youtube_dl/extractor/rai.py.orig 2020-11-16 21:04:10.000000000 +0000
25+++ youtube_dl/extractor/rai.py 24+++ youtube_dl/extractor/rai.py
26@@ -1,3 +1,4 @@ 25@@ -17,7 +17,6 @@ from ..utils import (
27+# coding: utf-8 26 int_or_none,
28 from __future__ import unicode_literals 
29  
30 import re 
31@@ -17,7 +18,6 @@ 
32 parse_duration, 27 parse_duration,
33 strip_or_none, 28 strip_or_none,
34 try_get, 
35- unescapeHTML, 29- unescapeHTML,
36 unified_strdate, 30 unified_strdate,
37 unified_timestamp, 31 unified_timestamp,
38 update_url_query, 32 update_url_query,
39@@ -30,6 +30,7 @@ class RaiBaseIE(InfoExtractor): 33@@ -30,6 +29,7 @@ class RaiBaseIE(InfoExtractor):
40 _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}' 34 _UUID_RE = r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
41 _GEO_COUNTRIES = ['IT'] 35 _GEO_COUNTRIES = ['IT']
42 _GEO_BYPASS = False 36 _GEO_BYPASS = False
43+ _BASE_URL = 'https://www.raiplay.it' 37+ _BASE_URL = 'https://www.raiplay.it'
44  38
45 def _extract_relinker_info(self, relinker_url, video_id): 39 def _extract_relinker_info(self, relinker_url, video_id):
46 if not re.match(r'https?://', relinker_url): 40 if not re.match(r'https?://', relinker_url):
47@@ -122,41 +123,19 @@ def _extract_subtitles(url, subtitle_url): 41@@ -122,27 +122,8 @@ class RaiBaseIE(InfoExtractor):
48  42
49  43
50 class RaiPlayIE(RaiBaseIE): 44 class RaiPlayIE(RaiBaseIE):
51- _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE 45- _VALID_URL = r'(?P<url>https?://(?:www\.)?raiplay\.it/.+?-(?P<id>%s)\.html)' % RaiBaseIE._UUID_RE
52+ _VALID_URL = r'(?P<url>(?P<base>https?://(?:www\.)?raiplay\.it/.+?-)(?P<id>%s)(?P<ext>\.(?:html|json)))' % RaiBaseIE._UUID_RE 46+ _VALID_URL = r'(?P<url>(?P<base>https?://(?:www\.)?raiplay\.it/.+?-)(?P<id>%s)(?P<ext>\.(?:html|json)))' % RaiBaseIE._UUID_RE
53 _TESTS = [{ 47 _TESTS = [{
54- 'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter', 48- 'url': 'http://www.raiplay.it/video/2016/10/La-Casa-Bianca-e06118bb-59a9-4636-b914-498e4cfd2c66.html?source=twitter',
55- 'md5': '340aa3b7afb54bfd14a8c11786450d76', 49- 'md5': '340aa3b7afb54bfd14a8c11786450d76',
56- 'info_dict': { 50- 'info_dict': {
57- 'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66', 51- 'id': 'e06118bb-59a9-4636-b914-498e4cfd2c66',
58- 'ext': 'mp4', 52- 'ext': 'mp4',
59- 'title': 'La Casa Bianca', 53- 'title': 'La Casa Bianca',
60- 'alt_title': 'S2016 - Puntata del 23/10/2016', 54- 'alt_title': 'S2016 - Puntata del 23/10/2016',
61- 'description': 'md5:a09d45890850458077d1f68bb036e0a5', 55- 'description': 'md5:a09d45890850458077d1f68bb036e0a5',
62- 'thumbnail': r're:^https?://.*\.jpg$', 56- 'thumbnail': r're:^https?://.*\.jpg$',
63- 'uploader': 'Rai 3', 57- 'uploader': 'Rai 3',
64- 'creator': 'Rai 3', 58- 'creator': 'Rai 3',
65- 'duration': 3278, 59- 'duration': 3278,
66- 'timestamp': 1477764300, 60- 'timestamp': 1477764300,
67- 'upload_date': '20161029', 61- 'upload_date': '20161029',
68- 'series': 'La Casa Bianca', 62- 'series': 'La Casa Bianca',
69- 'season': '2016', 63- 'season': '2016',
70- }, 64- },
 65- 'skip': 'This content is not available',
71- }, { 66- }, {
72 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html', 67 'url': 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
73 'md5': '8970abf8caf8aef4696e7b1f2adfc696', 68 'md5': '8970abf8caf8aef4696e7b1f2adfc696',
74 'info_dict': { 69 'info_dict': {
75 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391', 70@@ -166,10 +147,11 @@ class RaiPlayIE(RaiBaseIE):
76 'ext': 'mp4', 71 }]
77 'title': 'Report del 07/04/2014', 
78- 'alt_title': 'S2013/14 - Puntata del 07/04/2014', 
79- 'description': 'md5:f27c544694cacb46a078db84ec35d2d9', 
80+ 'alt_title': 'St 2013/14 - Espresso nel caffè - 07/04/2014 ', 
81+ 'description': 'md5:d730c168a58f4bb35600fc2f881ec04e', 
82 'thumbnail': r're:^https?://.*\.jpg$', 
83- 'uploader': 'Rai 5', 
84- 'creator': 'Rai 5', 
85+ 'uploader': 'Rai Gulp', 
86 'duration': 6160, 
87- 'series': 'Report', 
88- 'season_number': 5, 
89- 'season': '2013/14', 
90 }, 
91 'params': { 
92 'skip_download': True, 
93@@ -168,16 +147,15 @@ class RaiPlayIE(RaiBaseIE): 
94  72
95 def _real_extract(self, url): 73 def _real_extract(self, url):
96 mobj = re.match(self._VALID_URL, url) 74- url, video_id = re.match(self._VALID_URL, url).groups()
97- url, video_id = mobj.group('url', 'id') 75+ mobj = re.match(self._VALID_URL, url)
98+ url, base, video_id, ext = mobj.group('url', 'base', 'id', 'ext') 76+ url, base, video_id, ext = mobj.group('url', 'base', 'id', 'ext')
99  77
100 media = self._download_json( 78 media = self._download_json(
101- '%s?json' % url, video_id, 'Downloading video JSON') 79- url.replace('.html', '.json'), video_id, 'Downloading video JSON')
102+ '%s%s.json' % (base, video_id), video_id, 'Downloading video JSON') 80+ '%s%s.json' % (base, video_id), video_id, 'Downloading video JSON')
103  81
104 title = media['name'] 82 title = media['name']
105- 
106 video = media['video'] 
107  
108- relinker_info = self._extract_relinker_info(video['contentUrl'], video_id) 
109+ relinker_info = self._extract_relinker_info(video['content_url'], video_id) 
110 self._sort_formats(relinker_info['formats']) 
111  83
112 thumbnails = [] 84@@ -227,7 +209,7 @@ class RaiPlayLiveIE(RaiBaseIE):
113@@ -185,7 +163,7 @@ def _real_extract(self, url): 
114 for _, value in media.get('images').items(): 
115 if value: 
116 thumbnails.append({ 
117- 'url': value.replace('[RESOLUTION]', '600x400') 
118+ 'url': urljoin(RaiBaseIE._BASE_URL, value.replace('[RESOLUTION]', '600x400')) 
119 }) 
120  
121 timestamp = unified_timestamp(try_get( 
122@@ -225,7 +203,7 @@ class RaiPlayLiveIE(RaiBaseIE): 
123 'display_id': 'rainews24', 85 'display_id': 'rainews24',
124 'ext': 'mp4', 86 'ext': 'mp4',
125 'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 87 'title': 're:^Diretta di Rai News 24 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
126- 'description': 'md5:6eca31500550f9376819f174e5644754', 88- 'description': 'md5:6eca31500550f9376819f174e5644754',
127+ 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497', 89+ 'description': 'md5:4d00bcf6dc98b27c6ec480de329d1497',
128 'uploader': 'Rai News 24', 90 'uploader': 'Rai News 24',
129 'creator': 'Rai News 24', 91 'creator': 'Rai News 24',
130 'is_live': True, 92 'is_live': True,
131@@ -238,20 +216,32 @@ class RaiPlayLiveIE(RaiBaseIE): 93@@ -240,20 +222,32 @@ class RaiPlayLiveIE(RaiBaseIE):
132 def _real_extract(self, url): 94 def _real_extract(self, url):
133 display_id = self._match_id(url) 95 display_id = self._match_id(url)
134  96
135- webpage = self._download_webpage(url, display_id) 97- webpage = self._download_webpage(url, display_id)
136+ media = self._download_json( 98+ media = self._download_json(
137+ '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'dirette/' + display_id), 99+ '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'dirette/' + display_id),
138+ display_id, 'Downloading channel JSON') 100+ display_id, 'Downloading channel JSON')
139+ 101+
140+ title = media['name'] 102+ title = media['name']
141+ video = media['video'] 103+ video = media['video']
142+ video_id = media['id'].replace('ContentItem-', '') 104+ video_id = media['id'].replace('ContentItem-', '')
 105+
 106+ relinker_info = self._extract_relinker_info(video['content_url'], video_id)
 107+ self._sort_formats(relinker_info['formats'])
143  108
144- video_id = self._search_regex( 109- video_id = self._search_regex(
145- r'data-uniquename=["\']ContentItem-(%s)' % RaiBaseIE._UUID_RE, 110- r'data-uniquename=["\']ContentItem-(%s)' % RaiBaseIE._UUID_RE,
146- webpage, 'content id') 111- webpage, 'content id')
147+ relinker_info = self._extract_relinker_info(video['content_url'], video_id) 112-
148+ self._sort_formats(relinker_info['formats']) 
149  
150- return { 113- return {
151- '_type': 'url_transparent', 114- '_type': 'url_transparent',
152- 'ie_key': RaiPlayIE.ie_key(), 115- 'ie_key': RaiPlayIE.ie_key(),
153- 'url': 'http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id, 116- 'url': 'http://www.raiplay.it/dirette/ContentItem-%s.html' % video_id,
154+ info = { 117+ info = {
155 'id': video_id, 118 'id': video_id,
156 'display_id': display_id, 119 'display_id': display_id,
157+ 'title': self._live_title(title) if relinker_info.get( 120+ 'title': self._live_title(title) if relinker_info.get(
158+ 'is_live') else title, 121+ 'is_live') else title,
159+ 'alt_title': media.get('subtitle'), 122+ 'alt_title': media.get('subtitle'),
160+ 'description': media.get('description'), 123+ 'description': media.get('description'),
161+ 'uploader': strip_or_none(media.get('channel')), 124+ 'uploader': strip_or_none(media.get('channel')),
162+ 'creator': strip_or_none(media.get('editor')), 125+ 'creator': strip_or_none(media.get('editor')),
163+ 'duration': parse_duration(video.get('duration')), 126+ 'duration': parse_duration(video.get('duration')),
164 } 127 }
165  128
166+ info.update(relinker_info) 129+ info.update(relinker_info)
167+ return info 130+ return info
168+ 131+
169  132
170 class RaiPlayPlaylistIE(InfoExtractor): 133 class RaiPlayPlaylistIE(InfoExtractor):
171 _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+)' 134 _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+)'
172@@ -260,7 +250,7 @@ class RaiPlayPlaylistIE(InfoExtractor): 135@@ -262,7 +256,7 @@ class RaiPlayPlaylistIE(InfoExtractor):
173 'info_dict': { 136 'info_dict': {
174 'id': 'nondirloalmiocapo', 137 'id': 'nondirloalmiocapo',
175 'title': 'Non dirlo al mio capo', 138 'title': 'Non dirlo al mio capo',
176- 'description': 'md5:9f3d603b2947c1c7abb098f3b14fac86', 139- 'description': 'md5:9f3d603b2947c1c7abb098f3b14fac86',
177+ 'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b', 140+ 'description': 'md5:98ab6b98f7f44c2843fd7d6f045f153b',
178 }, 141 },
179 'playlist_mincount': 12, 142 'playlist_mincount': 12,
180 }] 143 }]
181@@ -268,21 +258,25 @@ class RaiPlayPlaylistIE(InfoExtractor): 144@@ -270,21 +264,25 @@ class RaiPlayPlaylistIE(InfoExtractor):
182 def _real_extract(self, url): 145 def _real_extract(self, url):
183 playlist_id = self._match_id(url) 146 playlist_id = self._match_id(url)
184  147
185- webpage = self._download_webpage(url, playlist_id) 148- webpage = self._download_webpage(url, playlist_id)
186+ media = self._download_json( 149+ media = self._download_json(
187+ '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id), 150+ '%s.json' % urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id),
188+ playlist_id, 'Downloading program JSON') 151+ playlist_id, 'Downloading program JSON')
189  152
190- title = self._html_search_meta( 153- title = self._html_search_meta(
191- ('programma', 'nomeProgramma'), webpage, 'title') 154- ('programma', 'nomeProgramma'), webpage, 'title')
192- description = unescapeHTML(self._html_search_meta( 155- description = unescapeHTML(self._html_search_meta(
193- ('description', 'og:description'), webpage, 'description')) 156- ('description', 'og:description'), webpage, 'description'))
194+ title = media['name'] 157+ title = media['name']
@@ -206,59 +169,52 @@ Shared upstream via: @@ -206,59 +169,52 @@ Shared upstream via:
206- video_id=RaiPlayIE._match_id(video_url))) 169- video_id=RaiPlayIE._match_id(video_url)))
207+ for cs in content_sets: 170+ for cs in content_sets:
208+ medias = self._download_json( 171+ medias = self._download_json(
209+ '%s/%s.json' % (urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id), cs), 172+ '%s/%s.json' % (urljoin(RaiBaseIE._BASE_URL, 'programmi/' + playlist_id), cs),
210+ cs, 'Downloading content set JSON') 173+ cs, 'Downloading content set JSON')
211+ for m in medias['items']: 174+ for m in medias['items']:
212+ video_url = urljoin(url, m['path_id']) 175+ video_url = urljoin(url, m['path_id'])
213+ entries.append(self.url_result( 176+ entries.append(self.url_result(
214+ video_url, ie=RaiPlayIE.ie_key(), 177+ video_url, ie=RaiPlayIE.ie_key(),
215+ video_id=RaiPlayIE._match_id(video_url))) 178+ video_id=RaiPlayIE._match_id(video_url)))
216  179
217 return self.playlist_result(entries, playlist_id, title, description) 180 return self.playlist_result(entries, playlist_id, title, description)
218  181
219@@ -316,7 +310,7 @@ class RaiIE(RaiBaseIE): 182@@ -330,19 +328,6 @@ class RaiIE(RaiBaseIE):
220 }, { 
221 # with ContentItem in og:url 
222 'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html', 
223- 'md5': '11959b4e44fa74de47011b5799490adf', 
224+ 'md5': '6865dd00cf0bbf5772fdd89d59bd768a', 
225 'info_dict': { 
226 'id': 'efb17665-691c-45d5-a60c-5301333cbb0c', 
227 'ext': 'mp4', 
228@@ -326,18 +320,6 @@ class RaiIE(RaiBaseIE): 
229 'duration': 2214, 
230 'upload_date': '20161103', 183 'upload_date': '20161103',
231 } 184 }
232- }, { 185 }, {
233- # drawMediaRaiTV(...) 186- # drawMediaRaiTV(...)
234- 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html', 187- 'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
235- 'md5': '2dd727e61114e1ee9c47f0da6914e178', 188- 'md5': '2dd727e61114e1ee9c47f0da6914e178',
236- 'info_dict': { 189- 'info_dict': {
237- 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af', 190- 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
238- 'ext': 'mp4', 191- 'ext': 'mp4',
239- 'title': 'Il pacco', 192- 'title': 'Il pacco',
240- 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a', 193- 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
241- 'thumbnail': r're:^https?://.*\.jpg$', 194- 'thumbnail': r're:^https?://.*\.jpg$',
242- 'upload_date': '20141221', 195- 'upload_date': '20141221',
243- }, 196- },
244 }, { 197- 'skip': 'This content is not available',
 198- }, {
245 # initEdizione('ContentItem-...' 199 # initEdizione('ContentItem-...'
246 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined', 200 'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
247@@ -349,17 +331,6 @@ class RaiIE(RaiBaseIE): 201 'info_dict': {
248 'upload_date': '20170401', 202@@ -354,18 +339,6 @@ class RaiIE(RaiBaseIE):
249 }, 203 },
250 'skip': 'Changes daily', 204 'skip': 'Changes daily',
251- }, { 205 }, {
252- # HDS live stream with only relinker URL 206- # HDS live stream with only relinker URL
253- 'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews', 207- 'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
254- 'info_dict': { 208- 'info_dict': {
255- 'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc', 209- 'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
256- 'ext': 'flv', 210- 'ext': 'flv',
257- 'title': 'EuroNews', 211- 'title': 'EuroNews',
258- }, 212- },
259- 'params': { 213- 'params': {
260- 'skip_download': True, 214- 'skip_download': True,
261- }, 215- },
262 }, { 216- 'skip': 'This content is available only in Italy',
 217- }, {
263 # HLS live stream with ContentItem in og:url 218 # HLS live stream with ContentItem in og:url
264 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html', 219 'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
 220 'info_dict': {