本帖最后由 xiaoxue 于 2019-3-4 20:51 编辑 . U# v+ Q6 D4 \" F; R
$ y7 ], C/ X4 \ s2 M n" W- # -*- coding: UTF-8 -*-
8 U4 b J$ b1 B0 n- g P - import requests
! X* } F* @( l$ p J$ g/ r0 Z - import os( b3 g8 b7 y* l+ e1 n
- import subprocess
- _% x1 k7 {7 h9 e+ z - import sys4 d) S6 G# L7 W. Q- Z2 q
- from tqdm import tqdm7 V% a% s* `$ W+ V6 A5 S% {
- import time,random. H( ]# \+ o3 {
- from HTMLParser import HTMLParser
) `, J2 q; V9 t: S+ v1 X+ h - import sys . A8 t- ]) p: m( ^7 Y/ a+ D; J
- reload(sys)
. ~- O/ R7 E7 U0 i& Y6 B4 @ - sys.setdefaultencoding('utf8') 2 r/ Z, L3 m0 {9 Q+ B5 r
( P3 C( T; U; h5 C- class WebRequest(object):) t2 t8 I) j0 N2 C, t! P# P
- def __init__(self, *args, **kwargs):; V) D- k8 y+ o$ @, z6 G3 F$ |
- pass; @. g8 ]3 \& r
- # O/ G# C2 J% X
- @property A) E: x& U# y- t2 N* K& `
- def user_agent(self):8 B, z8 m8 E1 t2 [3 `5 G
- """
8 u5 s( L/ c! x! E - return an User-Agent at random
) }8 `; F) J+ E - :return:5 J7 r$ A5 E$ z8 M0 L
- """& e, N! P- C. `4 M6 R5 |
- ua_list = [
7 ]2 E1 T4 p% j5 y - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
8 e. f: |$ Q/ `# m7 M7 `, L2 m - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122'," D4 @0 w% Y9 D8 L* h
- 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
0 \, j7 j: z4 C( O/ o - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',6 B Y# V* X9 ~5 ?
- 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
2 b& |! r( y0 `1 \- S - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
- p2 U0 r- Z# y. M/ P3 [* A# _ - 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',+ h2 X6 z! }" g( J$ p
- 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',! ?, S, w% W+ g2 N
- ]/ Q; v" A" y- ^8 k6 D; ^6 F
- return random.choice(ua_list)
) J6 G' S3 n5 }5 Y. j - . J0 u# X5 P3 R! m' F
- @property
8 ]: N3 q6 ^7 c8 w6 D2 ], u2 z4 O - def header(self):5 A$ z- e" [2 I7 o! Y2 M$ _
- """: f3 b6 l* q- J: l' h
- basic header5 K! N9 r# N7 S7 ?8 o
- :return:
( ~' N# y. v% X - """* W$ z4 r; a9 o; T+ m4 h* [
- return {'User-Agent': self.user_agent,. c6 l/ s: n4 C7 v _
- 'Accept': '*/*',( Q( b# a" b U9 w5 Y6 U: P5 u9 _8 O
- 'Connection': 'keep-alive',
! d1 k' c6 `# b$ N8 O" y' D - 'Accept-Language': 'zh-CN,zh;q=0.8'}
6 k" ]5 H; {4 c e- ^5 j1 k4 N - 2 `2 @1 A1 B N9 q
- def get(self, url, header=None, retry_time=5, timeout=30,
1 E7 l7 t2 k6 `" }7 U - retry_flag=list(), retry_interval=5, *args, **kwargs):3 Q6 N& r$ L3 X. n) b7 m' F
- """
0 d* w0 d: { R# H/ ] - get method
; }+ a# L7 f0 Y1 K8 L - :param url: target url
& y e# ]: R0 h+ A5 _/ A9 C) e- f - :param header: headers/ Y- b0 G6 p( p
- :param retry_time: retry time when network error2 C4 A! y+ K7 ]3 p n
- :param timeout: network timeout1 i9 t: B$ Q. B& f
- :param retry_flag: if retry_flag in content. do retry
/ b; P+ F8 Y2 r$ s - :param retry_interval: retry interval(second)5 T. J# M( q* m' z: s3 |
- :param args:# H# P+ g' W! M1 Y1 G
- :param kwargs:( v( C8 m5 W9 \7 |2 v
- :return:
; @% a# i2 H% v/ x- u+ y - """6 ^# A7 Q/ H V; R) l8 W7 C7 L
- headers = self.header
6 [9 s- c# ~+ O* n/ b - if header and isinstance(header, dict):
% V8 E! H* O; ~3 U2 j - headers.update(header)
* E8 @2 C' x) H0 |/ R9 | - while True:
* d5 u& W+ [( m, R9 W" V5 _ - try:
* @/ Y5 P4 b2 G: W, r& N - html = requests.get(url, headers=headers, timeout=timeout, stream=True)( M! |: d/ R: M: V* q, g
- print 'content size: %d' % len(html.content)- a0 e0 d- A! n* a# |" r
- if any(f in html.content for f in retry_flag):5 T! }. p+ {; e
- raise Exception7 n6 w1 a+ f8 H$ X( C+ B2 o' a ]
- if not html.content:% ^) I4 t5 V* N; E" ?, R# ?
- print 'content is Null,retry...~' + url
3 D0 U! Q: c2 N% }: ]& X2 B - raise Exception2 o" _" W0 V `" P# Y6 B% U$ h, g
- return html7 Q# N& w6 p" l) x x0 R' Y/ |
- except Exception as e:
; `' H* C4 R) }4 J: c& l4 ~% O - print(e)& R# M; Y$ g6 C0 `
3 U; N: t/ o5 D1 O. a" U- q% Z) x
: x7 {2 e4 S) ]4 @2 F! f2 t, @- def DownloadAudio(url, path, refer=None):, ]8 r6 T3 b8 {8 w/ O, H) t. R
- with open(path, 'wb') as handle:
5 R1 w6 K* l) j/ p. l2 f! n; _$ Y - if refer:* i0 Z) `% ] h( u$ n3 r0 _
- hdr = {
7 G/ |7 s4 [) @# k5 K - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',7 e4 @' C H( b9 S9 @8 U5 {7 }
- 'Referer': refer}( G2 p9 Y% V9 p
- else:% g5 o% m7 l! c/ F. _' A$ P6 r
- hdr = {7 n) M6 Z$ d8 v0 g
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',' O6 m% _1 q' Y9 ^+ }; W8 X
- 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',! i' o" ~) k1 @+ ]
- 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
" y: S. s0 G' f9 V - 'Accept-Encoding': 'none',1 ^+ D: i* b& g7 f( J5 |6 g4 t
- 'Accept-Language': 'en-US,en;q=0.8',
& C* C( k( Z- E) k9 a - 'Connection': 'keep-alive'}
3 D; z$ {1 m1 W; c2 X s3 v - wr = WebRequest()! L% o' U% L* q4 e# S5 U* p
- #print hdr
+ X+ n* d8 Y7 O/ }! \( ?. Q+ D - response = wr.get(url,header=hdr)8 V, n$ @* q# Y. e- r, [
- #html = requests.get('https://www.example.com', proxies={"http": "http://{}".format(proxy)})
" X8 s0 w$ N |% b( c" r0 z - #response = requests.get(url, stream=True, headers={'User-agent': 'Mozilla/5.0'})
% J; {* \' G. o3 Y - #print url4 I" q5 x$ Q6 ^7 g( F
- for block in tqdm(response.iter_content(),ascii=True, desc='ImageDownload'):
" O" @3 o+ H( _. e, j6 ^ - if not block:
$ p. s) r: m* i$ o, A2 g0 R - break
" o7 G' h' v4 U: u2 j4 l - handle.write(block)
/ Q f. n3 l! T0 @. O1 _
$ k+ G6 E1 Z" n0 C( z! \7 [- def translate(to_translate, to_langage="auto", langage="auto"):) Q/ {" {/ O% {1 Z9 K' z% N6 }. e
- '''Return the translation using google translate
- g+ |4 b: _# N" Z+ A* ` - you must shortcut the langage you define (French = fr, English = en, Spanish = es, etc...)9 d, R) r+ l9 \1 }. O* g
- if you don't define anything it will detect it or use english by default2 Z& O& D: @& l$ c9 w
- Example:/ I# [) R% }3 M
- print(translate("salut tu vas bien?", "en"))" U* B5 L) S+ q. p
- hello you alright?'''
* G" s& a W9 X1 n+ A. E {% j - agents = {'User-Agent':"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"}) ?5 \6 z$ \) i- b$ p/ C. Q
- before_trans = 'class="t0">'+ \1 z. y# ` {/ Q* C8 Q/ m( [
- link = "http://translate.google.com/m?ie=UTF-8&hl=%s&sl=%s&q=%s" % (to_langage, langage, to_translate.replace(" ", "+"))
5 N& x# c V* k4 f6 ^( N# @ - request = requests.get(link, headers=agents)
9 v, h8 R( ]( x. ` - page = request.content- _2 W4 h' [6 |$ x
- #print page
9 m; P1 |5 F) V1 J# W# f - result = page[page.find(before_trans)+len(before_trans):]
& _8 J+ |( G$ ]4 H, d0 m* h$ t - result = result.split("<")[0]8 O/ ]. r, ^% F( P$ W5 u, d
- return result9 ?6 a/ k+ b2 ^5 e
. c$ o( H/ x7 c* d$ [" A8 B- tsxt_sen = []5 N- q$ O7 h/ F6 W" N1 U
- from nltk.tokenize import sent_tokenize7 n$ D/ c5 x; S9 T& @7 H/ }: a
- def sentence_split(sentence): # 编写分句函数
) `) u/ B, v/ I$ y - text_sen = []
# ~+ N( u2 C# {" g - for s in sentence.split(','):! r( V: U/ w+ t' o+ G2 l
- # if '?' in s:
6 V) ~/ ^0 p' {* H0 k9 t" G/ {+ S1 A - # tsxt_sen.extend(s.split('?'))8 u8 F& U& v/ H1 |+ i) o
- # elif ',' in s:7 _( J% d6 X! A2 ]* r% e
- # tsxt_sen.extend(s.split(','))! \3 a1 D! k1 p' ]: k, e! Z
- # else:& w9 E. W8 Z# v, B2 s/ }
- text_sen.append(s)
* K3 }9 s$ _& u Z3 x9 e, J: k+ C - return tsxt_sen, N4 N) q2 |" K% D, n$ E( y0 L
- . X% H& n+ u7 C# _1 K( [
8 Y6 T# C( S; B8 {+ `" s- N! t- str = '''
N& Y8 Q$ ?) q; n* H - It has not quite been greeted with the enthusiasm of the bottles of Scotch in the novel Whisky Galore, but the arrival of a cargo of pineapples on the shores of northern Scotland has sent scavengers rushing to the beach./ N* A( W9 y. K9 Q
, v- O. o* x5 u) V% X- The fruit, which has appeared on Shetland and on beaches across the Western Isles, is thought to have come from several containers lost overboard in the Atlantic last October by cargo ship MV Lombok Strait., T* g1 ~! K2 k9 ?$ p% c4 B+ u
% I/ z" m9 b1 Z' q6 [& i- Shetland islanders have now shared photos of their fruitful haul, in a story that echoes the plot of Whisky Galore, the 1947 novel by Sir Compton Mackenzie, later made and remade for the big screen.% g7 e5 u5 A3 \& s0 ^, p
- ( r5 N/ J+ z* ?$ H( a3 Z f2 X
- The story tells of how the locals find their island awash with whisky after a ship loaded with liquor is wrecked off a fictional Scottish island. + }/ s0 ]6 F- Y5 l5 W3 Z
- # e% L) `2 X6 T9 H1 y% j6 L1 A
- The plot was based on the real-life salvage operations carried out by locals when the SS Politician, carrying 280,000 bottles of malt whisky, ran aground on Eriskay in 1943.( i. G( z: F" P( s
- ) f) W6 m$ C3 Z, b' ^" D4 z
- The arrival of the pineapples will be far less lucrative for beachcombers, and not as profitable, either, as the grounding of the cargo ship MSC Napoli off Branscombe beach in Devon in 2007.8 r* z' R* N* a- l8 X, O
$ e% V$ X: K4 [: ~# b) t; W3 @- Then scavengers ignored police advice and arrived in their droves to explore shipping containers, which contained motorbikes, pet food, wine barrels and anti-wrinkle cream.. @6 Q5 ]# n6 Q0 d* ]6 [) e: x" [
- '''0 o/ t' W& J2 V5 }- s' k7 C
! {' Q/ f" _( x6 r- 7 L% }6 U0 G- y& w; A/ B
- text = ''.join(str).strip().lstrip().rstrip().replace('\n',' ')/ X: I% p. y8 h* h
- with open('~/text.txt', 'wb') as handle:) E9 n6 c- j. X$ n- L1 i
- handle.write(text)
' I8 y9 d4 z3 N+ B9 [ - 2 I5 k# h% g C7 @6 m* M: R( ?
- str = str.replace('”','"')1 i" }6 e# g- B8 }0 q$ r; E
- str = str.replace('“','"')* `' Q' ^& i# F- |4 `' W
5 S P, J) k% K- ! X# _, T6 y l3 o
- def splitStr(str):9 x. z& p# C6 G1 d/ O- L! h1 S
- arr = str.split(' ')* l! J" U6 @+ R/ ~5 [
- sz = len(arr)
# v# p% |$ v. ]/ T- S - return [' '.join(arr[0:sz/2]),' '.join(arr[(sz/2):])]
1 ~$ N# H3 C' u' n! n: f' i8 v - ' _9 o% f3 A8 `/ j& I4 U* t9 g) l) s
- # x% T* F4 C# z0 [
- #print splitStr('with wording from North Korea about getting rid of its nuclear weapons and a guarantee from the United States that it would not interfere with the North’s regime or demand redress for human rights abuses.')" r0 j' P9 m- v9 G) Q
- * e! }- z; F8 t% f: i
- keyArr = []9 w* ~2 _; M, n2 S6 {+ q5 T; y! P$ X- h
- sent_tokenize_list = sent_tokenize(str)
% I$ N) Y N) Z7 X9 K - for s in sent_tokenize_list: |( N* Q) @, O% m
- sz = len(' '.join(s.split()))& P3 G5 O5 _) T' c& _6 A
- if sz <= 190:
. s+ ?; Y$ k$ t9 o - keyArr.append(' '.join(s.split()))
' b7 H6 X; U J; k" b' n8 i5 ~ - #print '[ '+' '.join(s.split())+' ]'
7 @$ S& L4 V' A. x+ m) U - else:' L, h: I7 X7 n! g) @2 O2 s
- for ses in s.split(','):. a. V' C/ ]8 q* v
- if len(ses) > 190:
! @* r( C0 q* Z6 ^3 @3 z6 [ - keyArr.extend(splitStr(ses))
3 w4 Y! i* [6 @ j, u1 F/ M& x - else:0 B5 ?# @9 V& o4 j: B" R! k
- keyArr.append(' '.join(ses.split()))+ a# X& k0 y/ d" O. u; R( I; T- @
5 P8 v6 M/ W! u! V( ^- & e; h! n) \, X9 {9 D) b( r+ j
- API_URL = "http://translate.google.com/translate_tts?ie=UTF-8&tl=en-us", m) x/ q8 q G1 n, ?
- #key = HTMLParser.unescape.__func__(HTMLParser, translate(str,'en'))1 Z3 d" }9 {% r+ }8 ?
- #keyArr = sentence_split(str)( W8 e4 X% K8 l& f" Z
- 0 q: Q/ s; I8 s
- ) z3 m& s4 K" y4 M
- #print([text[i:i+n] for i in xrange(0, len(text), n)])/ A: [% z4 I. ? G- |2 d1 l
- os.system('rm ~/tmp/audio/*.mp3')/ n7 @9 V; E6 n2 ^2 k0 A; g( z
- namei = 0
7 o4 N7 z2 e! t - for i in keyArr:
2 d8 s! s, W. A8 J - juzi = ' '.join(i.split())
4 v! o8 l0 x( o; [0 ^ - sz = len(juzi)
& c0 c$ J t- _/ H - link = API_URL+"&q="+juzi+"&client=tw-ob"
# q% ]3 d, y% Y+ T - print link
$ p, y& s2 `6 j" K2 ?( u) v5 ^ - path = "~/tmp/audio/%d.mp3" % namei
1 f9 O7 A( x) p4 q1 S4 l - #print path% [$ o: \! K9 N3 B+ S$ V
- DownloadAudio(link,path)- U% D: ]; ~9 p5 L, n% b
- os.system('file '+path)' b, F0 f1 D" g' n- z6 \
- time.sleep(1)) P# B: d0 e @4 b5 g
- namei = namei + 1
# l* m5 J/ C5 s, E/ {( w" l
乱七八糟拼一拼 就是个小玩意～～～～～ |
发表于 2018-8-28 13:42:09
楼主
发表于 2018-8-28 22:30:07
厉害啊，python高手啊！
发表于 2018-8-29 21:31:47
