1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
| #! /usr/bin/env/ python
# -*- coding: utf-8 -*-
#
#
from random import randint
import os
import time
import shutil
from multiprocessing import Process, Value, Manager
import sys
def testopen(nb, n1, n2, rc):
op = open
st = str
for i in range(0, nb):
out = op(rc + "couple_" + n1 + "_" + st(i) + '_' + n2 + "_" + st(i) +
".fasta", 'w')
src1 = op(n1 + '_' + st(i), 'r')
src2 = op(n2 + '_' + st(i), 'r')
out.write(src1.read() + src2.read())
src1.close()
src2.close()
out.close()
def testmp(lproxy, nb, n1, n2, rc):
op = open
st = str
while True:
try:
val = lproxy.pop()
except IndexError:
break
out = op(rc + "couple_" + n1 + "_" + val + '_' + n2 + "_" + val +
".fasta", 'w')
src1 = op(n1 + '_' + val, 'r')
src2 = op(n2 + '_' + val, 'r')
out.write(src1.read() + src2.read())
src1.close()
src2.close()
out.close()
def testmpmulti(lproxy, endtime, tclock, nb, n1, n2, rc):
op = open
st = str
while True:
try:
val = lproxy.pop()
except IndexError:
endtime.append(tclock())
break
out = op(rc + "couple_" + n1 + "_" + val + '_' + n2 + "_" + val +
".fasta", 'w')
src1 = op(n1 + '_' + val, 'r')
src2 = op(n2 + '_' + val, 'r')
out.write(src1.read() + src2.read())
src1.close()
src2.close()
out.close()
def inittest(nb, tests):
print('Grand ménage...')
print('Création des répertoires cibles')
for name in tests:
if os.path.isdir(name):
shutil.rmtree(name)
os.mkdir(name)
print('Suppression des fichiers')
for item in os.listdir(os.getcwd()):
if item.startswith('src'):
os.remove(item)
print('Création de 2 x ' + str(nb) + ' fichiers')
letters = 'acgt'
for index in range(0, nb):
src1 = ''
src2 = ''
for i in range(1, 88):
src1 += letters[randint(0, 3)]
src2 += letters[randint(0, 3)]
with open('src1_' + str(index), 'w') as f1:
f1.write(src1 + '\n')
with open('src2_' + str(index), 'w') as f2:
f2.write(src2 + '\n')
if __name__ == "__main__":
nb = 10000
tests = ['testopen', 'testmp', 'testmpmulti']
inittest(nb, tests)
print('Début des tests')
print('testopen:')
t1 = time.clock()
repcible = os.path.join(os.getcwd(), 'testopen') + os.sep
testopen(nb, 'src1', 'src2', repcible)
t2 = time.clock()
print(t2-t1)
print(len(os.listdir(repcible)))
print('testmp:')
t1 = time.clock()
lworkers = []
workers = 4
repcible = os.path.join(os.getcwd(), 'testmp') + os.sep
manager = Manager()
lproxy = manager.list()
for i in range(nb):
lproxy.append(str(i))
for w in range(workers):
lworkers.append(Process(target=testmp, args=(lproxy, nb, 'src1', 'src2',
repcible)))
for w in lworkers:
w.start()
while len(lproxy) > 0:
pass
t2 = time.clock()
print(t2-t1)
time.sleep(2)
print(len(os.listdir(repcible)))
print('testmpmulti:')
repcible = os.path.join(os.getcwd(), 'testmpmulti') + os.sep
manager = Manager()
endtime = manager.list()
lproxy = manager.list()
tclock = time.clock
for i in range(nb):
lproxy.append(str(i))
p1 = Process(target=testmpmulti, args=(lproxy, endtime, tclock, nb, 'src1',
'src2', repcible))
p2 = Process(target=testmpmulti, args=(lproxy, endtime, tclock, nb, 'src1',
'src2', repcible))
p3 = Process(target=testmpmulti, args=(lproxy, endtime, tclock, nb, 'src1',
'src2', repcible))
p4 = Process(target=testmpmulti, args=(lproxy, endtime, tclock, nb, 'src1',
'src2', repcible))
p1.start()
p2.start()
p3.start()
p4.start()
while any((p1.is_alive(), p2.is_alive(), p3.is_alive(), p4.is_alive())):
pass
print(max(endtime)) # A revoir
print(len(os.listdir(repcible))) |
Creer un grand nombre de fichier rapidement
Répondre avec citation
Envoyé par exeroc
. Je vais implémenter tes meilleurs solutions.
Partager