6

我正在使用舊線程發佈試圖解決相同問題的新代碼。 什麼構成安全的泡菜? this?Python分佈式計算(作品)

sock.py

from socket import socket 
from socket import AF_INET 
from socket import SOCK_STREAM 
from socket import gethostbyname 
from socket import gethostname 

class SocketServer: 
    def __init__(self, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.port = port 
    def listen(self, data): 
    self.sock.bind(("127.0.0.1", self.port)) 
    self.sock.listen(len(data)) 
    while data: 
     s = self.sock.accept()[0] 
     siz, dat = data.pop() 
     s.send(siz) 
     s.send(dat) 
     s.close() 

class Socket: 
    def __init__(self, host, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.sock.connect((host, port)) 
    def recv(self, size): 
    return self.sock.recv(size) 

pack.py

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 
from marshal import dumps as marshal_dumps 
from pickle import dumps as pickle_dumps 
from struct import pack as struct_pack 

class packer: 
    def __init__(self): 
    self.f = [] 
    def pack(self, what): 
    if type(what) is type(lambda:None): 
     self.f = [] 
     self.f.append(marshal_dumps(what.func_code)) 
     self.f.append(pickle_dumps(what.func_name)) 
     self.f.append(pickle_dumps(what.func_defaults)) 
     self.f.append(pickle_dumps(what.func_closure)) 
     self.f = pickle_dumps(self.f) 
     return (struct_pack('Q', len(self.f)), self.f) 

unpack.py

from types import FunctionType 
from pickle import loads as pickle_loads 
from marshal import loads as marshal_loads 
from struct import unpack as struct_unpack 
from struct import calcsize 

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 

class unpacker: 
    def __init__(self): 
    self.f = [] 
    self.fcompiled = lambda:None 
    self.sizeofsize = calcsize('Q') 
    def unpack(self, sock): 
    size = struct_unpack('Q', sock.recv(self.sizeofsize))[0] 
    self.f = pickle_loads(sock.recv(size)) 
    a = marshal_loads(self.f[0]) 
    b = globals() ## 
    c = pickle_loads(self.f[1]) 
    d = pickle_loads(self.f[2]) 
    e = pickle_loads(self.f[3]) 
    self.fcompiled = FunctionType(a, b, c, d, e) 
    return self.fcompiled 

test.py

from unpack import unpacker 
from pack import packer 
from sock import SocketServer 
from sock import Socket 
from threading import Thread 
from time import sleep 

count = 2 
port = 4446 

def f(): 
    print 42 

def server(): 
    ss = SocketServer(port) 
    pack = packer() 
    functions = [pack.pack(f) for nothing in range(count)] 
    ss.listen(functions) 

if __name__ == "__main__": 
    Thread(target=server).start() 
    sleep(1) 
    unpack = unpacker() 
    for nothing in range(count): 
    print unpack.unpack(Socket("127.0.0.1", port)) 

輸出:

<function f at 0x12917d0> 
<function f at 0x12915f0> 
+0

可以發佈一些示例代碼來測試腳本?謝謝 ! – 2011-06-02 10:19:40

+0

非常歡迎! – motoku 2011-06-02 10:36:09

+0

拋出的錯誤是什麼? – 2011-06-03 05:49:56

回答

2

ValueError: insecure string pickle當你的醃菜被損壞時引發。你確定你在一個sock.recv()(unpack.py)中接收了整個醃製對象嗎?

編輯:避免這種情況,你可以做任何大小(你Socket類必須支持接收至與緩衝區大小參數(即

class Socket: 
    def recv(self, bufsize): 
     return self.sock.recv(bufsize) 

)被調用):

import struct 

struct.pack('Q', len(pickled_list)) 
# Send it, and then send the pickled list. 

在接收機的程序:

import struct 

length = struct.unpack('Q', sock.recv(struct.calcsize('Q')))[0] 
pickled_list = sock.recv(length) 

'Q' 是一種unsigned long long。對於其他結構的東西,請參閱the struct module documentation

+0

謝謝。緩衝區太小。 – motoku 2011-06-04 18:56:09

+1

@Sean Pedersen查看我的編輯。 – 2011-06-04 19:49:10

4

我不認爲處理對象旨在通過網絡發送。查看multiprocessing/process.py中的第256行。

# We subclass bytes to avoid accidental transmission of auth keys over network. 

聽起來好像有一個很好的理由給我。如果你想分佈式計算,也許你應該看看library designed for that

+0

我只能使用本機庫。我的第一個選擇是循環導航安全措施,因爲實施是使用專用網絡。 – motoku 2011-06-03 06:37:06

+1

查看文檔,似乎'多處理'是'線程'的克隆,但在不同的進程,以避免全局解釋器鎖。我認爲你的選擇正在推出你自己的分佈式計算庫,或者說服管理開源的重要性。 ;) – 2011-06-03 06:58:06