ITPub博客

首页 > 数据库 > NoSQL > 【PyTHON】redis数据迁移

【PyTHON】redis数据迁移

原创 NoSQL 作者:风尘_NULL 时间:2016-03-25 09:36:58 0 删除 编辑
同事直接将redis2.8dump文件移到redis3.0上,结果出现了问题,于是本人就编写简单的数据迁移脚本,原理是通过遍历不同类型的数据,然后在插入redis3.0中
#!/usr/local/bin/python
#name:redisDataTans.py
try :
        from redis.sentinel import Sentinel
        from redis.client import StrictRedis
        from redis.client import Redis
except ImportError :
        print 'redis module import failed,please check the packege is installed'
//对于string类型的处理
def trans_string(from_obj,to_obj,key) :
        string_value=from_obj.get(key)
        to_obj.set(key,string_value)
//对于hash类型的处理
def trans_hash(from_obj,to_obj,key) :
        hash_value=from_obj.hgetall(key)
        to_obj.hmset(key,hash_value)
//对于列表的处理
def trans_list(from_obj,to_obj,key) :
        list_values=from_obj.lrange(key,0,-1)
        for list_value in list_values :
                to_obj.rpush(key,list_value)
//对于set的处理
def trans_set(from_obj,to_obj,key) :
        set_values=from_obj.smembers(key)
        for set_value in set_values :
                to_obj.sadd(key,set_value)
//对于zset的处理
def trans_zset(from_obj,to_obj,key) :
        try :
                zset_values=from_obj.zrange(name=key,start=0,end=-1,desc=True,withscores=True)
        except :
                print "error ,zrange get none values"
        for (name,score) in zset_values:
                to_obj.zadd(key,name,score)
//判断数据类型,选择相应的处理方法
def switch(data_type,from_object,to_object,key) :
        try :
                {"string":lambda from_object,to_object,key:trans_string(from_object,to_object,key),
                "list":lambda from_object,to_object,key:trans_list(from_object,to_object,key),
                "hash":lambda from_object,to_object,key:trans_hash(from_object,to_object,key),
                "set":lambda from_object,to_object,key:trans_set(from_object,to_object,key),
                "zset":lambda from_object,to_object,key:trans_zset(from_object,to_object,key)
                }[data_type](from_object,to_object,key)
        except KeyError :
                print '%s not found,please check it is a redis data type!'%data_type

if __name__=="__main__":
        //以下是测试用例
        connect_type=1
        socket_pair=[('10.16.147.195',26579)]
        serviceName='cluster63791'
        passwd='abc'
        dbname=15
        sentinel=Sentinel(sentinels=socket_pair)
        from_object=sentinel.master_for(service_name=serviceName,password=passwd,db=dbname)
        to_object=Redis(host='10.16.147.193',port=6379,db=6,password='cba')
        keys=from_object.keys('*')
        for key in keys :
                data_type=from_object.type(key)
                switch(data_type=data_type,from_object=from_object,to_object=to_object,key=key)

当然,我们可以做成命令行的形式
具体看下面脚本
#!/usr/local/bin/python
#name:redisTansfer.py
try :
        import argparse
        import redisDataTrans as transfer
except ImportError :
        print "argparse module import failed,please check the packege is installed"
if __name__=="__main__" :
        try :
                parse=argparse.ArgumentParser(prog="redisDataTrans",usage='%(prog)s [options]',description="this script transfer redis db from one to anthor,for example:ip1,db0=>ip2,db3")
                parse.add_argument('-sip','--source-ipaddress',required=True,help='the source server,which will transfer data to destination server')
                parse.add_argument('-dip','--destination-ipaddress',required=True,help='the destination server')
                parse.add_argument('-sport','--source-port',default=6379,help='the source redis port')
                parse.add_argument('-dport','--destination-port',default=6379,help='the destination redis port')
                parse.add_argument('-sdb','--source-database',default=0,type=int,choices=range(0,16),help='the redis dbname')
                parse.add_argument('-ddb','--destination-database',default=15,type=int,choices=range(0,16),help='the redis dbname')
                parse.add_argument('-sstnl','--issource-sentinel',action='store_true',default=False,help='the source redis use sentinel or not')
                parse.add_argument('-dstnl','--isdestination-sentinel',action='store_true',default=False,help='the desitination redis use sentinel or not')
                parse.add_argument('-spwd','--source-password',help='the source redis password')
                parse.add_argument('-dpwd','--destination-password',help='the source redis password')
                parse.add_argument('-scluster','--source-cluster',default='cluster63791',help='the sentinel cluster name')
                parse.add_argument('-dcluster','--destination-cluster',default='cluster63791',help='the sentinel cluster name')
                parse.add_argument('-keyvalue',default='*',help='the needed data to be transfer,for example:* or abc *')
        except :
                print 'paramter product error!'
                sys.exit()
        args=parse.parse_args()
        print args.issource_sentinel
        if args.issource_sentinel :
                print "create soure_sentinel object start ..."
                socket_pair=[(args.source_ipaddress,args.source_port)]
                serviceName=args.source_cluster
                passwd=args.source_password
                dbname=args.source_database
                sentinel=transfer.Sentinel(sentinels=socket_pair)
                try :
                        from_object=sentinel.master_for(service_name=serviceName,password=passwd,db=dbname)
                except :
                        print "create source_sentinel object failed!"
        else :
                try :
                        from_object=transfer.Redis(host=args.source_ipaddress,port=args.source_port,db=args.source_database,password=args.source_password)
                except :
                        print "create source_redis object failed!"
        if args.isdestination_sentinel :
                socket_pair=[(args.destination_ipaddress,args.destination_port)]
                serviceName=args.destination_cluster
                passwd=args.destination_password
                dbname=args.destination_database
                sentinel=transfer.Sentinel(sentinels=socket_pair)
                try :
                        to_object=sentinel.master_for(service_name=serviceName,password=passwd,db=dbname)
                except :
                        print "create destination_sentinel object failed!"
        else :
                try :
                        to_object=transfer.Redis(host=args.destination_ipaddress,port=args.destination_port,db=args.destination_database,password=args.destination_password)
                except :
                        print "create destination object failed!"

        keys=from_object.keys(args.keyvalue)
        for key in keys :
                data_type=from_object.type(key)
                transfer.switch(data_type=data_type,from_object=from_object,to_object=to_object,key=key)

来自 “ ITPUB博客 ” ,链接:http://blog.itpub.net/30221425/viewspace-2063525/,如需转载,请注明出处,否则将追究法律责任。

请登录后发表评论 登录
全部评论
我只是一名DBA

注册时间:2015-04-24

  • 博文量
    65
  • 访问量
    193415