基于七牛 Python SDK 写的一个同步脚本(支持批量下载)

#!/usr/bin/env python #-*- coding:utf-8 -*- #  # AUTHOR = "heqingpan" # AUTHOR_EMAIL = "[email protected]" # URL = "http://git.oschina.net/hqp/qiniu_sync" import qiniu from qiniu import Auth from qiniu import BucketManager import os import re import urllib2 import sys access_key = '' secret_key = '' bucket_name = '' bucket_domain = '' q = Auth (access_key, secret_key ) bucket = BucketManager (q ) basedir=os.path.realpath (os.path.dirname (__file__)) #同步目录 #basedir="" filename=__file__ ignore_paths=[filename,"{0}c".format (filename )] ignore_names=[".DS_Store",".git",".gitignore"] charset="utf8" diff_time=2*60 def list_all (bucket_name, bucket=None, prefix="", limit=100 ): rlist=[] if bucket is None: bucket = BucketManager (q ) marker = None eof = False while eof is False: ret, eof, info = bucket.list (bucket_name, prefix=prefix, marker=marker, limit=limit ) marker = ret.get ('marker', None ) for item in ret['items']: rlist.append (item["key"]) if eof is not True: # 错误处理 #print "error" pass return rlist def get_files (basedir="",fix="",rlist=None,ignore_paths=[],ignore_names=[]): if rlist is None: rlist=[] for subfile in os.listdir (basedir ): temp_path=os.path.join (basedir,subfile ) tp=os.path.join (fix,subfile ) if tp in ignore_names: continue if tp in ignore_paths: continue if os.path.isfile (temp_path ): rlist.append (tp ) elif os.path.isdir (temp_path ): get_files (temp_path,tp,rlist,ignore_paths,ignore_names ) return rlist def get_valid_key_files (subdir=""): basedir=subdir or basedir files = get_files (basedir=basedir,ignore_paths=ignore_paths,ignore_names=ignore_names ) return map (lambda f:(f.replace ("\\","/"),f ),files ) def sync (): qn_keys=list_all (bucket_name,bucket ) qn_set=set (qn_keys ) l_key_files=get_valid_key_files (basedir ) k2f={} update_keys=[] u_count=500 u_index=0 for k,f in l_key_files: k2f[k]=f str_k=k if isinstance (k,str ): k=k.decode (charset ) if k in qn_set: update_keys.append (str_k ) u_index+=1 if u_index > u_count: u_index-=u_count update_file (k2f,update_keys ) update_keys=[] else: # upload upload_file (k,os.path.join (basedir,f )) if update_keys: update_file (k2f,update_keys ) print "sync end" def update_file (k2f,ulist ): ops=qiniu.build_batch_stat (bucket_name,ulist ) rets,infos = bucket.batch (ops ) for i in xrange (len (ulist )): k=ulist[i] f=k2f.get (k ) ret=rets[i]["data"] size=ret.get ("fsize",None ) put_time = int (ret.get ("putTime")/10000000 ) local_size=os.path.getsize (f ) local_time=int (os.path.getatime (f )) if local_size==size: continue if put_time >= local_time - diff_time: # is new continue # update upload_file (k,os.path.join (basedir,f )) def upload_file (key,localfile ): print "upload_file:" print key token = q.upload_token (bucket_name, key ) mime_type = get_mime_type (localfile ) params = {'x:a': 'a'} progress_handler = lambda progress, total: progress ret, info = qiniu.put_file (token, key, localfile, params, mime_type, progress_handler=progress_handler ) def get_mime_type (path ): mime_type = "text/plain" return mime_type def down_file (key,basedir="",is_private=1,expires=3600 ): if isinstance (key,unicode ): key=key.encode (charset ) url = 'http://%s/%s' % (bucket_domain, key ) if is_private: url=q.private_download_url (url, expires=expires ) c=urllib2.urlopen (url ) fpath=key.replace ("/",os.sep ) savepath=os.path.join (basedir,fpath ) dir_=os.path.dirname (savepath ) if not os.path.isdir (dir_): os.makedirs (dir_) elif os.path.isfile (savepath ): os.remove (savepath ) f = file (savepath, 'wb') f.write (c.read ()) f.close () def down_all (prefix=""): import traceback for key in list_all (bucket_name,bucket,prefix=prefix ): try: down_file (key,basedir=basedir ) print "down:\t"+key except: print "error down:\t"+key print traceback.format_exc () print "down end" def main (): if len (sys.argv )>1: if sys.argv[1]=="down": prefix=len (sys.argv )>2 and sys.argv[2] or "" down_all (prefix=prefix ) return sync () if __name__=="__main__": main ()