Modify DecodingTask: fix digest check
[splitter-ng] / Splitter-ng / src / splitterng / DecodingTask.java
1 package splitterng;
2
3 import java.io.FileNotFoundException;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.io.OutputStream;
7 import java.nio.file.Files;
8 import java.nio.file.Path;
9 import java.security.MessageDigest;
10 import java.security.NoSuchAlgorithmException;
11 import java.util.ArrayList;
12 import java.util.Arrays;
13 import java.util.HashMap;
14 import java.util.Map;
15 import java.util.Map.Entry;
16
17 import splitterng.utils.ByteUtils;
18 import splitterng.utils.FileUtils;
19
20 public class DecodingTask extends CodingTask {
21         /**
22          * New Decoding Task
23          * 
24          * @param sourceFile
25          */
26
27         private OutputStream fileOutputStream = null;
28         private Map<Integer, InputStream> fileInputStreamMap = null;
29         ArrayList<Byte> dataAsList = null;
30
31         int rows;
32         private boolean[] erased;
33         private String digestFunction;
34         private HashMap<Integer, MessageDigest> fragmentDigests;
35         private HashMap<Integer, String> fragmentDigests_given;
36         private HashMap<Integer, String> fragmentDigests_failed;
37
38         public DecodingTask(Map<Integer, Path> fragmentPathsMap, Path outputFile,
39                         SplitterCallback callback, Map<String, Object> metadataMap) {
40                 this.isFileTask = true;
41                 this.filePath = outputFile;
42                 this.fragmentPathsMap = fragmentPathsMap;
43                 this.rows = fragmentPathsMap.size();
44                 this.metadataMap = metadataMap;
45                 this.callback = callback;
46
47                 erased = new boolean[fragmentPathsMap.size()];
48
49                 readMetadataMap();
50                 if (digestFunction != null){
51                         digestCheck(); // remove from fragmentPathsMap when not OK
52                 }
53                 
54         }
55
56         public DecodingTask(Map<Integer, byte[]> dataFragmentMap,
57                         SplitterCallback callback, Map<String, Object> metadataMap) {
58                 this.isFileTask = false;
59                 this.dataFragmentMap = dataFragmentMap;
60                 this.rows = this.dataFragmentMap.size();
61                 this.metadataMap = metadataMap;
62                 this.callback = callback;
63
64                 erased = new boolean[this.dataFragmentMap.size()];
65
66                 readMetadataMap();
67
68         }
69
70         private void readMetadataMap() {
71                 if (!this.metadataMap.containsKey("fragmentsize_coded")) {
72                         throw new IllegalArgumentException(
73                                         "Missing Metadata: fragmentsize_coded");
74                 } else {
75                         fragmentsize = (int) this.metadataMap.get("fragmentsize_coded");
76                 }
77
78                 if (!this.metadataMap.containsKey("originalSize")) {
79                         throw new IllegalArgumentException("Missing Metadata: originalSize");
80                 } else {
81                         originalSize = (long) this.metadataMap.get("originalSize");
82                 }
83
84                 if (!this.metadataMap.containsKey("truncate")) {
85                         throw new IllegalArgumentException("Missing Metadata: truncate");
86                 } else {
87                         truncate = (int) this.metadataMap.get("truncate");
88                 }
89
90                 if (!this.metadataMap.containsKey("readins")) {
91                         throw new IllegalArgumentException("Missing Metadata: readins");
92                 } else {
93                         readins = (long) this.metadataMap.get("readins");
94                 }
95
96                 if (this.metadataMap.containsKey("digest")) {
97                         digestFunction = (String) this.getMetadata("digest");
98                         fragmentDigests_given = new HashMap<Integer, String>();
99                         fragmentDigests_failed = new HashMap<Integer, String>();
100                         for (Entry<String, Object> e : metadataMap.entrySet()) {
101                                 if (e.getKey().startsWith("digest_")) {
102                                         fragmentDigests_given.put(
103                                                         Integer.parseInt(e.getKey().replaceAll("digest_",
104                                                                         "")), (String) e.getValue());
105                                 }
106                         }
107
108                 }
109
110         }
111
112         public boolean hasDataToRead() {
113                 if (readin == readins) {
114                         return false;
115                 } else {
116                         return true;
117                 }
118         }
119
120         public byte[][] readData() throws IOException {
121                 if (isFileTask) {
122                         return readDataFromFiles();
123                 } else {
124                         return readDataFromByteArray();
125                 }
126         }
127
128         private byte[][] readDataFromByteArray() {
129                 // Init data if size is not correct
130                 // Eventuell in den Constructor?
131                 if (workingDataFragemnts == null
132                                 || (workingDataFragemnts.length != fragmentPathsMap.size())
133                                 || (workingDataFragemnts[0].length != fragmentsize)) {
134                         workingDataFragemnts = new byte[rows][fragmentsize];
135                 }
136                 // is this the first readin?
137                 if (readin == 0) {
138                         setMetadata("time_decode_start", System.currentTimeMillis());
139                 }
140
141                 for (int i = 0; i < dataFragmentMap.size(); i++) {
142                         byte[] byteArray = dataFragmentMap.get(i);
143                         if (byteArray == null | (fragmentDigests_failed.containsKey(i))) {
144                                 // data[i]=null;
145                                 // Fill row with 0
146                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
147                         } else {
148                                 // read in data.
149                                 System.arraycopy(byteArray, 0, workingDataFragemnts[i], 0,
150                                                 byteArray.length);
151                                 readBytesTotal += byteArray.length;
152                         }
153                 }
154                 readin++;
155
156                 if (readin == readins) {
157                         setMetadata("restoredFromBytes", readBytesTotal);
158                 }
159
160                 return workingDataFragemnts;
161
162         }
163
164         private void digestCheck() {
165                 if (isFileTask) {
166                         for (Entry<Integer, Path> entry : fragmentPathsMap.entrySet()) {
167                                 if (entry.getValue() != null) {
168                                         try {
169                                                 String fileDigest = FileUtils.hashFile(entry.getValue().toFile(),digestFunction);
170                                                 if (fileDigest.equals(fragmentDigests_given.get(entry.getKey()))){
171                                                         setMetadata("digestcheck_" + entry.getKey(), "OK");
172                                                 }
173                                                 else{
174                                                         setMetadata("digestcheck_" + entry.getKey(), "MISMATCH: "+fileDigest );
175                                                         fragmentPathsMap.put(entry.getKey(), null); // Ignore corrupt files
176                                                 }
177                                         } catch (FileNotFoundException e) {
178                                                 setMetadata("digestcheck_" + entry.getKey(), e.getMessage());
179                                         } catch (NoSuchAlgorithmException e) {
180                                                 fragmentDigests=null;
181                                                 digestFunction=null;
182                                                 setMetadata("digest_error", e.getMessage());
183                                                 break;
184                                         } catch (IOException e) {
185                                                 setMetadata("digestcheck_" + entry.getKey(), e.getMessage());
186                                         }
187                                 }
188                         }
189                 }
190         }
191
192         private void updateDigests(byte[][] fragments) {
193                 if (digestFunction != null) {
194                         for (int i = 0; i < fragments.length; i++) {
195                                 if (!fragmentDigests.containsKey(i)) {
196                                         try {
197                                                 fragmentDigests.put(i,
198                                                                 MessageDigest.getInstance(digestFunction));
199                                         } catch (NoSuchAlgorithmException e) {
200                                                 e.printStackTrace();
201                                                 // TODO Besseres Handling, wenn Algorithm nicht
202                                                 // vorhanden
203                                                 fragmentDigests = null;
204                                                 digestFunction = null;
205                                                 setMetadata("digest_error", e.getMessage());
206                                                 break;
207                                         }
208                                 }
209
210                                 fragmentDigests.get(i).update(fragments[i]);
211                         }
212                 }
213
214         }
215
216         private void finishDigest() {
217                 if (digestFunction != null) {
218                         for (int i : fragmentDigests.keySet()) {
219                                 setMetadata("digest_" + i,
220                                                 ByteUtils.toHexString(fragmentDigests.get(i).digest()));
221                         }
222                 }
223         }
224
225         /**
226          * @return Array of ByteArrays. If there are erasures, ByteArray is replaced
227          *         by null
228          * @throws IOException
229          */
230         private byte[][] readDataFromFiles() throws IOException {
231
232                 // Init data if size is not correct
233                 // Eventuell in den Constructor?
234                 if (workingDataFragemnts == null
235                                 || (workingDataFragemnts.length != fragmentPathsMap.size())
236                                 || (workingDataFragemnts[0].length != fragmentsize)) {
237                         workingDataFragemnts = new byte[rows][fragmentsize];
238                 }
239
240                 // is this the first readin?
241                 if (readin == 0 || fileInputStreamMap == null) {
242                         setMetadata("time_decode_start", System.currentTimeMillis());
243
244                         // Create FileInputStreams
245                         fileInputStreamMap = new HashMap<Integer, InputStream>();
246                         for (Entry<Integer, Path> p : fragmentPathsMap.entrySet()) {
247                                 if (p.getValue() == null || ((fragmentDigests_failed != null) && (fragmentDigests_failed.containsKey(p.getKey())))   ) {
248                                         fileInputStreamMap.put(p.getKey(), null);
249                                 } else {
250                                         fileInputStreamMap.put(p.getKey(),
251                                                         Files.newInputStream(p.getValue()));
252                                 }
253                         }
254                 }
255
256                 for (int i = 0; i < fileInputStreamMap.size(); i++) {
257                         InputStream stream = fileInputStreamMap.get(Integer.valueOf(i));
258                         if (stream == null) {
259                                 // data[i]=null;
260                                 // Fill row with 0
261                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
262                         } else {
263                                 // read in data.
264                                 int readBytes = stream.read(workingDataFragemnts[i]);
265                                 readBytesTotal += readBytes;
266                         }
267                 }
268                 readin++;
269                 if (readin == readins) {
270                         for (InputStream s : fileInputStreamMap.values()) {
271                                 if (s != null) {
272                                         s.close();
273                                 }
274                         }
275                         setMetadata("restoredFromBytes", readBytesTotal);
276                 }
277
278                 return workingDataFragemnts;
279         }
280
281         public boolean[] getErased() {
282                 if (isFileTask) {
283                         for (int i = 0; i < fragmentPathsMap.size(); i++) {
284                                 if ((fragmentPathsMap.get(i) == null) || ((fragmentDigests_failed != null) && (fragmentDigests_failed.containsKey(i)))) {
285                                         erased[i] = true;
286                                 } else {
287                                         erased[i] = false;
288                                 }
289                         }
290                 } else {
291                         for (int i = 0; i < dataFragmentMap.size(); i++) {
292                                 if (dataFragmentMap.get(i) == null|| ((fragmentDigests_failed != null) && (fragmentDigests_failed.containsKey(i)))) {
293                                         erased[i] = true;
294                                 } else {
295                                         erased[i] = false;
296                                 }
297                         }
298
299                 }
300                 return erased;
301
302         }
303
304         public void writeDecodedData(byte[][] fragments) throws IOException {
305                 if (isFileTask) {
306                         writeDecodedDataToFiles(fragments);
307                 } else {
308                         writeDecodedDataToByteArray(fragments);
309                 }
310         }
311
312         private void writeDecodedDataToByteArray(byte[][] fragments) {
313                 // is this the first Fragment?
314                 if (writeout == 0) {
315                         dataAsList = new ArrayList<Byte>();
316                 }
317
318                 for (int i = 0; i < fragments.length; i++) {
319                         if ((writtenBytesTotal + fragments[i].length) <= originalSize) {
320                                 // write
321                                 dataAsList.ensureCapacity(fragments[i].length);
322                                 for (byte b : fragments[i]) {
323                                         dataAsList.add(b);
324                                 }
325                                 writtenBytesTotal += fragments[i].length;
326                         } else if ((writtenBytesTotal + fragments[i].length) > originalSize) {
327                                 // write partly
328                                 dataAsList
329                                                 .ensureCapacity((int) (originalSize - writtenBytesTotal));
330                                 for (int j = 0; j < (int) (originalSize - writtenBytesTotal); j++) {
331                                         dataAsList.add(fragments[i][j]);
332                                 }
333                                 writtenBytesTotal = originalSize;
334                         } else if ((writtenBytesTotal) == originalSize) {
335                                 // write nothing
336                         }
337                 }
338                 writeout++;
339
340                 if (writtenBytesTotal == originalSize && writeout == readins) {
341                         setMetadata("time_decode_end", System.currentTimeMillis());
342                         setMetadata("restoredBytes", writtenBytesTotal);
343
344                         // TODO Das hier ist ineffizient!!!
345                         this.data = new byte[dataAsList.size()];
346                         for (int j = 0; j < dataAsList.size(); j++) {
347                                 this.data[j] = dataAsList.get(j);
348                         }
349                         setCodingSuccessfull(true);
350                         setCodingFinished(true);
351                 }
352
353         }
354
355         private void writeDecodedDataToFiles(byte[][] fragments) throws IOException {
356                 // is this the first Fragment?
357                 if (writeout == 0 || fileOutputStream == null) {
358                         Files.createFile(filePath);
359                         fileOutputStream = Files.newOutputStream(filePath);
360                 }
361
362                 for (int i = 0; i < fragments.length; i++) {
363                         if ((writtenBytesTotal + fragments[i].length) <= originalSize) {
364                                 // write
365                                 fileOutputStream.write(fragments[i]);
366                                 writtenBytesTotal += fragments[i].length;
367                         } else if ((writtenBytesTotal + fragments[i].length) > originalSize) {
368                                 // write partly
369                                 fileOutputStream.write(fragments[i], 0,
370                                                 (int) (originalSize - writtenBytesTotal));
371                                 writtenBytesTotal = originalSize;
372                         } else if ((writtenBytesTotal) == originalSize) {
373                                 // write nothing
374                         }
375                 }
376                 writeout++;
377
378                 if (writtenBytesTotal == originalSize && writeout == readins) {
379                         setMetadata("time_decode_end", System.currentTimeMillis());
380                         fileOutputStream.close();
381                         setMetadata("restoredBytes", writtenBytesTotal);
382                         setCodingSuccessfull(true);
383                         setCodingFinished(true);
384                 }
385
386         }
387
388         @Override
389         public boolean isFileTask() {
390                 return this.isFileTask;
391         }
392
393 }