3f4c7098eed2266ad15b2c9c4025128c548d7054
[splitter-ng] / Splitter-ng / src / splitterng / DecodingTask.java
1 package splitterng;
2
3 import java.io.FileNotFoundException;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.io.OutputStream;
7 import java.nio.file.Files;
8 import java.nio.file.Path;
9 import java.security.MessageDigest;
10 import java.security.NoSuchAlgorithmException;
11 import java.util.ArrayList;
12 import java.util.Arrays;
13 import java.util.HashMap;
14 import java.util.Map;
15 import java.util.Map.Entry;
16
17 import splitterng.utils.ByteUtils;
18 import splitterng.utils.FileUtils;
19
20 public class DecodingTask extends CodingTask {
21         /**
22          * New Decoding Task
23          * 
24          * @param sourceFile
25          */
26
27         private OutputStream fileOutputStream = null;
28         private Map<Integer, InputStream> fileInputStreamMap = null;
29         ArrayList<Byte> dataAsList = null;
30
31         int rows;
32         private boolean[] erased;
33         private String digestFunction;
34         private HashMap<Integer, MessageDigest> fragmentDigests;
35         private HashMap<Integer, String> fragmentDigests_given;
36         private HashMap<Integer, String> fragmentDigests_failed;
37
38         public DecodingTask(Map<Integer, Path> fragmentPathsMap, Path outputFile,
39                         SplitterCallback callback, Map<String, Object> metadataMap) {
40                 this.isFileTask = true;
41                 this.filePath = outputFile;
42                 this.fragmentPathsMap = fragmentPathsMap;
43                 this.rows = fragmentPathsMap.size();
44                 this.metadataMap = metadataMap;
45                 this.callback = callback;
46
47                 erased = new boolean[fragmentPathsMap.size()];
48
49                 readMetadataMap();
50                 digestCheck(); // remove from fragmentPathsMap when not OK
51                 
52         }
53
54         public DecodingTask(Map<Integer, byte[]> dataFragmentMap,
55                         SplitterCallback callback, Map<String, Object> metadataMap) {
56                 this.isFileTask = false;
57                 this.dataFragmentMap = dataFragmentMap;
58                 this.rows = this.dataFragmentMap.size();
59                 this.metadataMap = metadataMap;
60                 this.callback = callback;
61
62                 erased = new boolean[this.dataFragmentMap.size()];
63
64                 readMetadataMap();
65
66         }
67
68         private void readMetadataMap() {
69                 if (!this.metadataMap.containsKey("fragmentsize_coded")) {
70                         throw new IllegalArgumentException(
71                                         "Missing Metadata: fragmentsize_coded");
72                 } else {
73                         fragmentsize = (int) this.metadataMap.get("fragmentsize_coded");
74                 }
75
76                 if (!this.metadataMap.containsKey("originalSize")) {
77                         throw new IllegalArgumentException("Missing Metadata: originalSize");
78                 } else {
79                         originalSize = (long) this.metadataMap.get("originalSize");
80                 }
81
82                 if (!this.metadataMap.containsKey("truncate")) {
83                         throw new IllegalArgumentException("Missing Metadata: truncate");
84                 } else {
85                         truncate = (int) this.metadataMap.get("truncate");
86                 }
87
88                 if (!this.metadataMap.containsKey("readins")) {
89                         throw new IllegalArgumentException("Missing Metadata: readins");
90                 } else {
91                         readins = (long) this.metadataMap.get("readins");
92                 }
93
94                 if (this.metadataMap.containsKey("digest")) {
95                         digestFunction = (String) this.getMetadata("digest");
96                         fragmentDigests_given = new HashMap<Integer, String>();
97                         fragmentDigests_failed = new HashMap<Integer, String>();
98                         for (Entry<String, Object> e : metadataMap.entrySet()) {
99                                 if (e.getKey().startsWith("digest_")) {
100                                         fragmentDigests_given.put(
101                                                         Integer.parseInt(e.getKey().replaceAll("digest_",
102                                                                         "")), (String) e.getValue());
103                                 }
104                         }
105
106                 }
107
108         }
109
110         public boolean hasDataToRead() {
111                 if (readin == readins) {
112                         return false;
113                 } else {
114                         return true;
115                 }
116         }
117
118         public byte[][] readData() throws IOException {
119                 if (isFileTask) {
120                         return readDataFromFiles();
121                 } else {
122                         return readDataFromByteArray();
123                 }
124         }
125
126         private byte[][] readDataFromByteArray() {
127                 // Init data if size is not correct
128                 // Eventuell in den Constructor?
129                 if (workingDataFragemnts == null
130                                 || (workingDataFragemnts.length != fragmentPathsMap.size())
131                                 || (workingDataFragemnts[0].length != fragmentsize)) {
132                         workingDataFragemnts = new byte[rows][fragmentsize];
133                 }
134                 // is this the first readin?
135                 if (readin == 0) {
136                         setMetadata("time_decode_start", System.currentTimeMillis());
137                 }
138
139                 for (int i = 0; i < dataFragmentMap.size(); i++) {
140                         byte[] byteArray = dataFragmentMap.get(i);
141                         if (byteArray == null | (fragmentDigests_failed.containsKey(i))) {
142                                 // data[i]=null;
143                                 // Fill row with 0
144                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
145                         } else {
146                                 // read in data.
147                                 System.arraycopy(byteArray, 0, workingDataFragemnts[i], 0,
148                                                 byteArray.length);
149                                 readBytesTotal += byteArray.length;
150                         }
151                 }
152                 readin++;
153
154                 if (readin == readins) {
155                         setMetadata("restoredFromBytes", readBytesTotal);
156                 }
157
158                 return workingDataFragemnts;
159
160         }
161
162         private void digestCheck() {
163                 if (isFileTask) {
164                         for (Entry<Integer, Path> entry : fragmentPathsMap.entrySet()) {
165                                 if (entry.getValue() != null) {
166                                         try {
167                                                 String fileDigest = FileUtils.hashFile(entry.getValue().toFile(),digestFunction);
168                                                 if (fileDigest.equals(fragmentDigests_given.get(entry.getKey()))){
169                                                         setMetadata("digestcheck_" + entry.getKey(), "OK");
170                                                 }
171                                                 else{
172                                                         setMetadata("digestcheck_" + entry.getKey(), "MISMATCH: "+fileDigest );
173                                                         fragmentPathsMap.put(entry.getKey(), null); // Ignore corrupt files
174                                                 }
175                                         } catch (FileNotFoundException e) {
176                                                 setMetadata("digestcheck_" + entry.getKey(), e.getMessage());
177                                         } catch (NoSuchAlgorithmException e) {
178                                                 fragmentDigests=null;
179                                                 digestFunction=null;
180                                                 setMetadata("digest_error", e.getMessage());
181                                                 break;
182                                         } catch (IOException e) {
183                                                 setMetadata("digestcheck_" + entry.getKey(), e.getMessage());
184                                         }
185                                 }
186                         }
187                 }
188         }
189
190         private void updateDigests(byte[][] fragments) {
191                 if (digestFunction != null) {
192                         for (int i = 0; i < fragments.length; i++) {
193                                 if (!fragmentDigests.containsKey(i)) {
194                                         try {
195                                                 fragmentDigests.put(i,
196                                                                 MessageDigest.getInstance(digestFunction));
197                                         } catch (NoSuchAlgorithmException e) {
198                                                 e.printStackTrace();
199                                                 // TODO Besseres Handling, wenn Algorithm nicht
200                                                 // vorhanden
201                                                 fragmentDigests = null;
202                                                 digestFunction = null;
203                                                 setMetadata("digest_error", e.getMessage());
204                                                 break;
205                                         }
206                                 }
207
208                                 fragmentDigests.get(i).update(fragments[i]);
209                         }
210                 }
211
212         }
213
214         private void finishDigest() {
215                 if (digestFunction != null) {
216                         for (int i : fragmentDigests.keySet()) {
217                                 setMetadata("digest_" + i,
218                                                 ByteUtils.toHexString(fragmentDigests.get(i).digest()));
219                         }
220                 }
221         }
222
223         /**
224          * @return Array of ByteArrays. If there are erasures, ByteArray is replaced
225          *         by null
226          * @throws IOException
227          */
228         private byte[][] readDataFromFiles() throws IOException {
229
230                 // Init data if size is not correct
231                 // Eventuell in den Constructor?
232                 if (workingDataFragemnts == null
233                                 || (workingDataFragemnts.length != fragmentPathsMap.size())
234                                 || (workingDataFragemnts[0].length != fragmentsize)) {
235                         workingDataFragemnts = new byte[rows][fragmentsize];
236                 }
237
238                 // is this the first readin?
239                 if (readin == 0 || fileInputStreamMap == null) {
240                         setMetadata("time_decode_start", System.currentTimeMillis());
241
242                         // Create FileInputStreams
243                         fileInputStreamMap = new HashMap<Integer, InputStream>();
244                         for (Entry<Integer, Path> p : fragmentPathsMap.entrySet()) {
245                                 if (p.getValue() == null | (fragmentDigests_failed.containsKey(p.getKey()))) {
246                                         fileInputStreamMap.put(p.getKey(), null);
247                                 } else {
248                                         fileInputStreamMap.put(p.getKey(),
249                                                         Files.newInputStream(p.getValue()));
250                                 }
251                         }
252                 }
253
254                 for (int i = 0; i < fileInputStreamMap.size(); i++) {
255                         InputStream stream = fileInputStreamMap.get(Integer.valueOf(i));
256                         if (stream == null) {
257                                 // data[i]=null;
258                                 // Fill row with 0
259                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
260                         } else {
261                                 // read in data.
262                                 int readBytes = stream.read(workingDataFragemnts[i]);
263                                 readBytesTotal += readBytes;
264                         }
265                 }
266                 readin++;
267                 if (readin == readins) {
268                         for (InputStream s : fileInputStreamMap.values()) {
269                                 if (s != null) {
270                                         s.close();
271                                 }
272                         }
273                         setMetadata("restoredFromBytes", readBytesTotal);
274                 }
275
276                 return workingDataFragemnts;
277         }
278
279         public boolean[] getErased() {
280                 if (isFileTask) {
281                         for (int i = 0; i < fragmentPathsMap.size(); i++) {
282                                 if ((fragmentPathsMap.get(i) == null) | (fragmentDigests_failed.containsKey(i)) ) {
283                                         erased[i] = true;
284                                 } else {
285                                         erased[i] = false;
286                                 }
287                         }
288                 } else {
289                         for (int i = 0; i < dataFragmentMap.size(); i++) {
290                                 if (dataFragmentMap.get(i) == null| (fragmentDigests_failed.containsKey(i))) {
291                                         erased[i] = true;
292                                 } else {
293                                         erased[i] = false;
294                                 }
295                         }
296
297                 }
298                 return erased;
299
300         }
301
302         public void writeDecodedData(byte[][] fragments) throws IOException {
303                 if (isFileTask) {
304                         writeDecodedDataToFiles(fragments);
305                 } else {
306                         writeDecodedDataToByteArray(fragments);
307                 }
308         }
309
310         private void writeDecodedDataToByteArray(byte[][] fragments) {
311                 // is this the first Fragment?
312                 if (writeout == 0) {
313                         dataAsList = new ArrayList<Byte>();
314                 }
315
316                 for (int i = 0; i < fragments.length; i++) {
317                         if ((writtenBytesTotal + fragments[i].length) <= originalSize) {
318                                 // write
319                                 dataAsList.ensureCapacity(fragments[i].length);
320                                 for (byte b : fragments[i]) {
321                                         dataAsList.add(b);
322                                 }
323                                 writtenBytesTotal += fragments[i].length;
324                         } else if ((writtenBytesTotal + fragments[i].length) > originalSize) {
325                                 // write partly
326                                 dataAsList
327                                                 .ensureCapacity((int) (originalSize - writtenBytesTotal));
328                                 for (int j = 0; j < (int) (originalSize - writtenBytesTotal); j++) {
329                                         dataAsList.add(fragments[i][j]);
330                                 }
331                                 writtenBytesTotal = originalSize;
332                         } else if ((writtenBytesTotal) == originalSize) {
333                                 // write nothing
334                         }
335                 }
336                 writeout++;
337
338                 if (writtenBytesTotal == originalSize && writeout == readins) {
339                         setMetadata("time_decode_end", System.currentTimeMillis());
340                         setMetadata("restoredBytes", writtenBytesTotal);
341
342                         // TODO Das hier ist ineffizient!!!
343                         this.data = new byte[dataAsList.size()];
344                         for (int j = 0; j < dataAsList.size(); j++) {
345                                 this.data[j] = dataAsList.get(j);
346                         }
347                         setCodingSuccessfull(true);
348                         setCodingFinished(true);
349                 }
350
351         }
352
353         private void writeDecodedDataToFiles(byte[][] fragments) throws IOException {
354                 // is this the first Fragment?
355                 if (writeout == 0 || fileOutputStream == null) {
356                         Files.createFile(filePath);
357                         fileOutputStream = Files.newOutputStream(filePath);
358                 }
359
360                 for (int i = 0; i < fragments.length; i++) {
361                         if ((writtenBytesTotal + fragments[i].length) <= originalSize) {
362                                 // write
363                                 fileOutputStream.write(fragments[i]);
364                                 writtenBytesTotal += fragments[i].length;
365                         } else if ((writtenBytesTotal + fragments[i].length) > originalSize) {
366                                 // write partly
367                                 fileOutputStream.write(fragments[i], 0,
368                                                 (int) (originalSize - writtenBytesTotal));
369                                 writtenBytesTotal = originalSize;
370                         } else if ((writtenBytesTotal) == originalSize) {
371                                 // write nothing
372                         }
373                 }
374                 writeout++;
375
376                 if (writtenBytesTotal == originalSize && writeout == readins) {
377                         setMetadata("time_decode_end", System.currentTimeMillis());
378                         fileOutputStream.close();
379                         setMetadata("restoredBytes", writtenBytesTotal);
380                         setCodingSuccessfull(true);
381                         setCodingFinished(true);
382                 }
383
384         }
385
386         @Override
387         public boolean isFileTask() {
388                 return this.isFileTask;
389         }
390
391 }