Discussion :

[airod]

bonjour, je viens de commencer a expérimenter le multiprocessing.
j'ai codé une classe qui a pour but de déplacer dans plusieurs dossiers des fichiers se trouvant dans un autre dossier, et de créer un Process de traitement pour chacun des fichiers déplacés.
Mon test unitaire de ma classe fonctionne.
Mais lorsque je l'implémente dans mon programme principale, j'ai bien la première phase qui s'effectue bien (déplacement des fichiers) mais j'ai une erreur lors de la création des Process.
ci joint l'erreur :

Exception in thread Thread-2:
Traceback (most recent call last):
File "C:\Python25\lib\threading.py", line 486, in __bootstrap_inner
self.run()
File "C:\Python25\lib\threading.py", line 446, in run
self.__target(*self.__args, **self.__kwargs)
File "D:\WORKDIRECTORY\PRJ_PYTHON\Atlantis\P001_.py", line 209, in traitement_cmd
cmdWKD.dispatchCommande()
File "D:\WORKDIRECTORY\PRJ_PYTHON\Atlantis\cmd.py", line 61, in dispatchCommande
p.start()
File "C:\Python25\Lib\site-packages\multiprocessing\process.py", line 109, in start
self._popen = Popen(self)
File "C:\Python25\Lib\site-packages\multiprocessing\forking.py", line 244, in __init__
dump(process_obj, to_child, HIGHEST_PROTOCOL)
File "C:\Python25\Lib\site-packages\multiprocessing\forking.py", line 167, in dump
ForkingPickler(file, protocol).dump(obj)
File "C:\Python25\Lib\pickle.py", line 224, in dump
self.save(obj)
File "C:\Python25\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python25\Lib\pickle.py", line 419, in save_reduce
save(state)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Python25\Lib\pickle.py", line 681, in _batch_setitems
save(v)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\site-packages\multiprocessing\forking.py", line 40, in dispatcher
self.save_reduce(obj=obj, *rv)
File "C:\Python25\Lib\pickle.py", line 401, in save_reduce
save(args)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 548, in save_tuple
save(element)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 725, in save_inst
save(stuff)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Python25\Lib\pickle.py", line 681, in _batch_setitems
save(v)
File "C:\Python25\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python25\Lib\pickle.py", line 419, in save_reduce
save(state)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Python25\Lib\pickle.py", line 681, in _batch_setitems
save(v)
File "C:\Python25\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python25\Lib\pickle.py", line 419, in save_reduce
save(state)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 649, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Python25\Lib\pickle.py", line 686, in _batch_setitems
save(v)
File "C:\Python25\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python25\Lib\pickle.py", line 396, in save_reduce
save(cls)
File "C:\Python25\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python25\Lib\pickle.py", line 748, in save_global
(obj, module, name))
PicklingError: Can't pickle <type 'PySwigObject'>: it's not found as __builtin__.PySwigObject

voici ma classe :

Code :

Sélectionner tout - Visualiser dans une fenêtre à part

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import util
import os
import shutil
import multiprocessing
import time
 
 
class ACommande:
    def __init__(self,parent=None,path=os.curdir,proc=1):
        self.path=path
        self.parent=parent
        self.proc=int(proc)
 
 
    def listProcess(self,path):
        '''retourne True si le dossier Process est libre'''
        a=util.Listage(path=path)
        if len(a.listfile('cma')['index'])==0:
            return True
        else:
            return False
 
    def nextCommande(self,path):
        '''retourne apres avoir teste, le nom du fichiers de commande a traiter'''
        a=util.Listage(path=path)
        liste=a.listfile('cma')
        return liste[liste['index'][0]]+'.cma'
 
    def createCommande(self,fullcmd):
        '''declenche la commande'''
 
        a=0
        while a<20:
            time.sleep(2)
            a+=2
            print a
 
 
    def dispatchCommande(self):
        ''' envoie la commande dans le dossier Process libre'''
        process=[]
        dirProc=[]
        for a in range(0,self.proc):
            dirProc.append('P'+str(a))
        #print dirProc
        path=self.path+'/PROCESS'
        if not os.path.isdir(path):
            os.makedirs(path)
        queue=multiprocessing.Queue()
        for i in dirProc:
            if not os.path.isdir(path+'/'+i):
                os.makedirs(path+'/'+i)
 
            if self.listProcess(path+'/'+i) and not self.listProcess(path):
                cmd=self.nextCommande(path)
                queue.put(path+'/'+i+'/'+cmd)
                shutil.move(path+'/'+cmd,path+'/'+i+'/'+cmd)
 
                process.append(multiprocessing.Process(target=self.createCommande,args=(queue,)))
        for p in process:
            p.start()
        for p in process:
            p.join()
 
 
 
 
if __name__ == "__main__":
    a=ACommande(path='C:/Hotfolders2',proc=2)
    while 1:
        a.dispatchCommande()
    raw_input()

et la fonction de mon programme principale (interface en wxPython):

Code :

Sélectionner tout - Visualiser dans une fenêtre à part

1
2
3
4
5
6
7
8
9
10
11
12
13
def traitement_cmd(self):
        PROCESS=self.dicDatas['workDirectory']['directory']+'/PROCESS'
        QUEUE=self.dicDatas['workDirectory']['directory']+'/QUEUE'
        WKD=self.dicDatas['workDirectory']['directory']
        cmdWKD=cmd.ACommande(parent=self,path=self.dicDatas['workDirectory']['directory'],proc=3)
        while 1:
            time.sleep(0.25)
            if self.ON==True:
                print cmdWKD.listProcess(PROCESS),cmdWKD.listProcess(QUEUE)
                if cmdWKD.listProcess(PROCESS) and not  cmdWKD.listProcess(QUEUE):
                    CMD=cmdWKD.nextCommande(QUEUE)
                    shutil.move(QUEUE+'/'+CMD,PROCESS+'/'+CMD)
                    cmdWKD.dispatchCommande()

malgré mes tests de modifications je n'arrive pas a résoudre le problème.
est ce que qqu'un peut m'aider?
merci d'avance