1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
from qiniu import Auth
from qiniu import BucketManager
import requests
import os
access_key = "aaa"
secret_key = "bbb"
q = Auth(access_key, secret_key)
bucket = BucketManager(q)
prefix = None
limit = 1000
delimiter = None
local_save_path = "H:/Picture/qiniu_pic"
local_map = {}
bucket_map = {
"hexo-blog" : "http://domain01.com",
'mywiki' : 'http://domain02.com',
'mywiki-2' : 'http://domain03.com',
'mywiki-3' : 'http://domain04.com',
}
def download_bucket(bucketName, bucketDomain, marker, cnt):
save_dir = os.path.join(local_save_path, bucketName)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if cnt == 0:
print(
"------------ start download bucket:[%s], domain:[%s]"
% (bucketName, bucketDomain)
)
ret, eof, info = bucket.list(bucketName, prefix, marker, limit, delimiter)
if ret == None:
print("--- Error, ret is None!", ret)
return
for i in ret.get("items"):
webfile = i["key"]
if (
local_map.get(bucketName) != None
and local_map.get(bucketName).get(webfile) != None
):
continue
base_url = "%s/%s" % (bucketDomain, webfile)
print("--- downloading: %s" % base_url)
private_url = q.private_download_url(base_url, expires=100)
r = requests.get(private_url)
if r.content:
cnt += 1
file = open(os.path.join(save_dir, webfile), "wb")
file.write(r.content)
file.flush()
file.close()
retMarker = ret.get("marker")
if retMarker != None:
download_bucket(bucketName, bucketDomain, retMarker, cnt)
else:
print(
"------------ over, bucket:[%s], domain:[%s], count:[%d]"
% (bucketName, bucketDomain, cnt)
)
print()
def collect_local_exist(bucketName):
save_dir = os.path.join(local_save_path, bucketName)
if not os.path.exists(save_dir):
return
local_map[bucketName] = {}
for root, dirs, files in os.walk(save_dir):
for file in files:
local_map[bucketName][file] = True
def main():
for (k, v) in bucket_map.items():
collect_local_exist(k)
download_bucket(k, v, None, 0)
if __name__ == "__main__":
main()
|
