Modify EncodingTask: add codecid to default filename
[splitter-ng] / Splitter-ng / src / splitterng / EncodingTask.java
1 package splitterng;
2
3 import java.io.IOException;
4 import java.io.InputStream;
5 import java.io.OutputStream;
6 import java.nio.file.Files;
7 import java.nio.file.Path;
8 import java.security.MessageDigest;
9 import java.security.NoSuchAlgorithmException;
10 import java.util.ArrayList;
11 import java.util.Arrays;
12 import java.util.HashMap;
13 import java.util.Map;
14
15 import splitterng.utils.ByteUtils;
16
17 public class EncodingTask extends CodingTask {
18
19         private InputStream fileInputStream = null;
20         private Map<Integer, OutputStream> fileOutputStreamMap = null;
21         private Map<Integer, ArrayList<Byte>> dataFragmentAsListMap = null;
22         private String digestFunction;
23         private Map<Integer, MessageDigest> fragmentDigests = null;
24         
25         public EncodingTask(Path file, Path outputDir, SplitterCallback callback,
26                         Map<String, Object> metadataMap) throws IOException {
27                 this.isFileTask = true;
28                 this.filePath = file;
29                 this.outputDir = outputDir;
30                 this.metadataMap = metadataMap;
31                 this.callback = callback;
32                 this.originalSize = Files.size(this.filePath);
33                 this.truncate = 0;
34
35                 readMetadataMap();
36
37                 this.setMetadata("originalSize", originalSize);
38
39         }
40
41         public EncodingTask(byte[] data, SplitterCallback callback,
42                         Map<String, Object> metadataMap) throws IOException {
43                 this.isFileTask = false;
44                 this.data = data;
45                 this.metadataMap = metadataMap;
46                 this.callback = callback;
47                 this.originalSize = this.data.length;
48                 this.truncate = 0;
49
50                 readMetadataMap();
51
52                 this.setMetadata("originalSize", originalSize);
53
54         }
55
56         private void readMetadataMap() {
57                 if (!this.metadataMap.containsKey("fragmentsize")) {
58                         throw new IllegalArgumentException("Missing Metadata: blockSize");
59                 } else {
60                         this.fragmentsize = (int) this.getMetadata("fragmentsize");
61                 }
62                 if (this.metadataMap.containsKey("digest")) {
63                         digestFunction = (String)this.getMetadata("digest");
64                         fragmentDigests = new HashMap<Integer, MessageDigest>();
65                         
66                 }
67         }
68
69         @Override
70         public boolean hasDataToRead() {
71                 if (this.readBytesTotal < this.originalSize) {
72                         return true;
73                 }
74                 return false;
75         }
76
77         public byte[][] readData(int rows) throws IOException {
78                 if (isFileTask) {
79                         return readDataFromFile(rows);
80                 } else {
81                         return readDataFromByteArray(rows);
82                 }
83
84         }
85
86         private byte[][] readDataFromByteArray(int rows) {
87                 if (this.workingDataFragemnts == null) {
88                         setMetadata("time_encode_start", System.currentTimeMillis());
89                 }
90                 if (this.workingDataFragemnts == null
91                                 || (this.workingDataFragemnts.length != rows)
92                                 || (this.workingDataFragemnts[0].length != this.fragmentsize)) {
93                         this.workingDataFragemnts = new byte[rows][this.fragmentsize];
94                 }
95
96                 for (int i = 0; i < rows; i++) {
97                         int from = (int) (i * this.readin);
98                         if (from <= this.originalSize) {
99                                 workingDataFragemnts[i] = Arrays.copyOfRange(this.data, from,
100                                                 from + this.fragmentsize);
101                                 if (from + this.fragmentsize > this.originalSize) {
102                                         readBytesTotal += this.fragmentsize
103                                                         - (from + this.fragmentsize - this.originalSize);
104                                         truncate += from + this.fragmentsize - this.originalSize;
105                                 } else {
106                                         readBytesTotal += this.fragmentsize;
107                                 }
108                         } else {
109                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
110                                 truncate += this.fragmentsize;
111                         }
112                 }
113                 readin++;
114
115                 if (readBytesTotal == originalSize) {
116                         setMetadata("truncate", truncate);
117                         setMetadata("readins", readin);
118                 }
119
120                 return workingDataFragemnts;
121
122         }
123
124         private byte[][] readDataFromFile(int rows) throws IOException {
125                 if (this.fileInputStream == null) {
126                         setMetadata("time_encode_start", System.currentTimeMillis());
127                         this.fileInputStream = Files.newInputStream(filePath);
128                 }
129                 if (this.workingDataFragemnts == null
130                                 || (this.workingDataFragemnts.length != rows)
131                                 || (this.workingDataFragemnts[0].length != this.fragmentsize)) {
132                         this.workingDataFragemnts = new byte[rows][this.fragmentsize];
133                 }
134                 // System.out.println(Thread.currentThread() + " lese block: " + readin
135                 // + "\tFragmente:" + rows + "  Fragmentlänge: " + this.fragmentsize
136                 // + "   Bytes: " + (rows*this.fragmentsize >(1024*1024) ?
137                 // (rows*this.fragmentsize)/(1024*1024)+"M" : rows*this.fragmentsize));
138                 for (int i = 0; i < rows; i++) {
139                         int readBytes = fileInputStream.read(workingDataFragemnts[i]);
140                         if (readBytes == -1) { // no data read, fill row with 0
141                                 Arrays.fill(workingDataFragemnts[i], (byte) 0);
142                                 truncate += this.fragmentsize;
143                         } else if (readBytes < this.fragmentsize) { // row not full, fill
144                                                                                                                 // row with 0
145                                 readBytesTotal += readBytes;
146                                 Arrays.fill(workingDataFragemnts[i], readBytes,
147                                                 this.fragmentsize - 1, (byte) 0);
148                                 truncate += this.fragmentsize - readBytes;
149                         } else {
150                                 readBytesTotal += readBytes;
151                         }
152                 }
153                 readin++;
154
155                 if (readBytesTotal == originalSize) {
156                         fileInputStream.close();
157                         setMetadata("truncate", truncate);
158                         setMetadata("readins", readin);
159                 }
160
161                 return workingDataFragemnts;
162
163         }
164
165         public void writeEncodedData(byte[][] fragments) throws IOException {
166                 if (fragmentDigests != null){
167                         updateDigests(fragments);
168                 }
169                 if (isFileTask) {
170                         writeEncodedDataToFiles(fragments);
171                 } else {
172                         writeEncodedDataToByteArrays(fragments);
173                 }
174         }
175
176         private void updateDigests(byte[][] fragments) {
177                 if (digestFunction != null){
178                 for (int i = 0; i < fragments.length; i++) {
179                         if (! fragmentDigests.containsKey(i)){
180                                 try {
181                                         fragmentDigests.put(i, MessageDigest.getInstance(digestFunction));
182                                 } catch (NoSuchAlgorithmException e) {
183                                         e.printStackTrace();
184                                         //TODO Besseres Handling, wenn Algorithm nicht vorhanden
185                                         fragmentDigests=null;
186                                         digestFunction=null;
187                                         setMetadata("digesterror", e.getMessage());
188                                         break;
189                                 }
190                         }
191                         
192                         fragmentDigests.get(i).update(fragments[i]);
193                 }
194                 }
195                 
196         }
197         private void finishDigest() {
198                 if (digestFunction != null){
199                         for (int i : fragmentDigests.keySet()) {
200                                 setMetadata("digest_"+i, ByteUtils.toHexString(fragmentDigests.get(i).digest()));
201                         }
202                 }
203         }
204         private void writeEncodedDataToByteArrays(byte[][] fragments) {
205                 // is this the first Fragment?
206                 if (writeout == 0) {
207                         setMetadata("fragmentsize_coded", fragments[0].length);
208
209                         dataFragmentAsListMap = new HashMap<Integer, ArrayList<Byte>>();
210                 }
211
212                 for (int i = 0; i < fragments.length; i++) {
213                         if (dataFragmentAsListMap.get(i) == null) {
214                                 dataFragmentAsListMap.put(i, new ArrayList<Byte>(
215                                                 fragments[i].length));
216                         }
217
218                         ArrayList<Byte> fragementByteList = dataFragmentAsListMap.get(i);
219                         fragementByteList.ensureCapacity(fragments[i].length);
220                         for (byte b : fragments[0]) {
221                                 fragementByteList.add(b);
222                         }
223                         writtenBytesTotal += fragments[i].length;
224                 }
225                 writeout++;
226
227                 if (readBytesTotal == originalSize && writeout == readin) {
228                         setMetadata("time_encode_end", System.currentTimeMillis());
229                         setMetadata("encodedSize", writtenBytesTotal);
230                         
231                         dataFragmentMap = new HashMap<Integer, byte[]>();
232                         
233                         //TODO Das hier ist ineffizient!!!
234                         for (int i = 0; i < dataFragmentAsListMap.size(); i++) {
235                                 ArrayList<Byte> arrayList = dataFragmentAsListMap.get(i);
236                                 byte[] byteArray = new byte[arrayList.size()];
237                                 for (int j = 0; j < arrayList.size(); j++) {
238                                         byteArray[j] = arrayList.get(j);
239                                 }
240                                 dataFragmentMap.put(i, byteArray);
241                         }
242                         finishDigest();
243                         setCodingSuccessfull(true);
244                         setCodingFinished(true);
245                 }
246
247         }
248
249
250
251         private void writeEncodedDataToFiles(byte[][] fragments) throws IOException {
252
253                 // is this the first Fragment?
254                 if (writeout == 0) {
255                         setMetadata("fragmentsize_coded", fragments[0].length);
256
257                         fragmentPathsMap = new HashMap<Integer, Path>();
258                         fileOutputStreamMap = new HashMap<Integer, OutputStream>();
259                         for (int i = 0; i < fragments.length; i++) {
260                                 Path newfile = this.outputDir.resolve(filePath.getFileName()
261                                                 + "#"
262                                                 + getMetadata("codecid")
263                                                 + "#" 
264                                                 + hashCode()
265                                                 + "#" + i);
266                                 // System.out.println("Erzeuge Fragmentdatei: "+newfile);
267                                 fragmentPathsMap.put(i, Files.createFile(newfile));
268                                 fileOutputStreamMap.put(i, Files.newOutputStream(newfile));
269                         }
270                 }
271                 // System.out.println(Thread.currentThread()+ " schreibe block: "+
272                 // writeout + "\t" + fragments.length + "  " +fragments[0].length);
273
274                 for (int i = 0; i < fragments.length; i++) {
275                         fileOutputStreamMap.get(i).write(fragments[i]);
276                         writtenBytesTotal += fragments[i].length;
277                 }
278                 writeout++;
279
280                 if (readBytesTotal == originalSize && writeout == readin) {
281                         setMetadata("time_encode_end", System.currentTimeMillis());
282                         for (OutputStream s : fileOutputStreamMap.values()) {
283                                 s.close();
284                         }
285                         setMetadata("encodedSize", writtenBytesTotal);
286                         finishDigest();
287                         setCodingSuccessfull(true);
288                         setCodingFinished(true);
289                 }
290
291         }
292
293         @Override
294         public boolean isFileTask() {
295                 return this.isFileTask;
296         }
297
298 }