From dc23c6a445dd8796ff2b6f0d91a9c19cd4c91d6c Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Tue, 18 Sep 2018 10:11:44 +0200 Subject: [PATCH] Optimize memory pool and revert algo --- .../keepass/database/load/ImporterV4.java | 42 +++++--- .../save/PwDbInnerHeaderOutputV4.java | 20 +--- .../keepass/database/save/PwDbV4Output.java | 102 +++++++++++------- .../com/kunzisoft/keepass/utils/MemUtil.java | 83 +++++++++++++- 4 files changed, 176 insertions(+), 71 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 221aece5d..83d49428c 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -23,7 +23,6 @@ import com.kunzisoft.keepass.R; import com.kunzisoft.keepass.crypto.CipherFactory; import com.kunzisoft.keepass.crypto.PwStreamCipherFactory; import com.kunzisoft.keepass.crypto.engine.CipherEngine; -import com.kunzisoft.keepass.database.BinaryPool; import com.kunzisoft.keepass.database.ITimeLogger; import com.kunzisoft.keepass.database.PwCompressionAlgorithm; import com.kunzisoft.keepass.database.PwDatabase; @@ -211,7 +210,10 @@ public class ImporterV4 extends Importer { while(true) { if (!ReadInnerHeader(lis, header)) break; } + } + private String getUnusedCacheFileName() { + return String.valueOf(db.getBinPool().findUnusedKey()); } private boolean ReadInnerHeader(LEDataInputStream lis, PwDbHeaderV4 header) throws IOException { @@ -241,15 +243,14 @@ public class ImporterV4 extends Importer { byte flag = lis.readBytes(1)[0]; boolean protectedFlag = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != PwDbHeaderV4.KdbxBinaryFlags.None; + int byteLength = size - 1; // Read in a file - BinaryPool binaryPool = db.getBinPool(); - int binaryKey = binaryPool.findUnusedKey(); - File file = new File(streamDir, String.valueOf(binaryKey)); + File file = new File(streamDir, getUnusedCacheFileName()); try (FileOutputStream outputStream = new FileOutputStream(file)) { - lis.readBytes(size - 1, outputStream::write); + lis.readBytes(byteLength, outputStream::write); } - ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, size -1); - binaryPool.add(protectedBinary); + ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, byteLength); + db.getBinPool().add(protectedBinary); break; default: @@ -288,9 +289,9 @@ public class ImporterV4 extends Importer { } private static final long DEFAULT_HISTORY_DAYS = 365; - + private boolean readNextNode = true; - private Stack ctxGroups = new Stack(); + private Stack ctxGroups = new Stack<>(); private PwGroupV4 ctxGroup = null; private PwEntryV4 ctxEntry = null; private String ctxStringName = null; @@ -336,7 +337,7 @@ public class ImporterV4 extends Importer { ctxGroups.clear(); KdbContext ctx = KdbContext.Null; - + readNextNode = true; while (true) { @@ -1027,7 +1028,19 @@ public class ImporterV4 extends Importer { return new ProtectedString(false, ReadString(xpp)); } - + + private ProtectedBinary createProtectedBinaryFromData(boolean protection, byte[] data) throws IOException { + if (data.length > MemUtil.BUFFER_SIZE_BYTES) { + File file = new File(streamDir, getUnusedCacheFileName()); + try (FileOutputStream outputStream = new FileOutputStream(file)) { + outputStream.write(data); + } + return new ProtectedBinary(protection, file, data.length); + } else { + return new ProtectedBinary(protection, data); + } + } + private ProtectedBinary ReadProtectedBinary(XmlPullParser xpp) throws XmlPullParserException, IOException { String ref = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrRef); if (ref != null) { @@ -1045,8 +1058,9 @@ public class ImporterV4 extends Importer { byte[] buf = ProcessNode(xpp); - if ( buf != null ) - return new ProtectedBinary(true, buf); + if ( buf != null ) { + createProtectedBinaryFromData(true, buf); + } String base64 = ReadString(xpp); if ( base64.length() == 0 ) @@ -1058,7 +1072,7 @@ public class ImporterV4 extends Importer { data = MemUtil.decompress(data); } - return new ProtectedBinary(false, data); + return createProtectedBinaryFromData(false, data); } private String ReadString(XmlPullParser xpp) throws IOException, XmlPullParserException { diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java index b54eb6d92..a362ab7c0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java @@ -23,9 +23,9 @@ import com.kunzisoft.keepass.database.PwDatabaseV4; import com.kunzisoft.keepass.database.PwDbHeaderV4; import com.kunzisoft.keepass.database.security.ProtectedBinary; import com.kunzisoft.keepass.stream.LEDataOutputStream; +import com.kunzisoft.keepass.utils.MemUtil; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; public class PwDbInnerHeaderOutputV4 { @@ -60,26 +60,8 @@ public class PwDbInnerHeaderOutputV4 { los.writeInt((int) protectedBinary.length() + 1); // TODO verify los.write(flag); - /* MemUtil.readBytes(protectedBinary.getData(), - (int) protectedBinary.length(), buffer -> los.write(buffer)); - */ - - byte[] buffer = new byte[3 * 256]; // TODO buffer generalize - InputStream fileInputStream = protectedBinary.getData(); // TODO Nullable - // To create the last buffer who is smaller - long numberOfFullBuffer = protectedBinary.length() / buffer.length; - long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; - int read = 0; - //if (protectedBinary.length() > 0) { - while (read < protectedBinary.length()) { - // Create the last smaller buffer - if (read >= sizeOfFullBuffers) - buffer = new byte[(int) (protectedBinary.length() % buffer.length)]; - read += fileInputStream.read(buffer, 0, buffer.length); - los.write(buffer); - } } los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.EndOfHeader); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 810e5ba9e..d697ae517 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -394,9 +394,8 @@ public class PwDbV4Output extends PwDbOutput { if (!isHistory) { writeList(PwDatabaseV4XML.ElemHistory, entry.getHistory(), true); - } else { - assert(entry.sizeOfHistory() == 0); } + // else entry.sizeOfHistory() == 0 xml.endTag(null, PwDatabaseV4XML.ElemEntry); } @@ -424,54 +423,85 @@ public class PwDbV4Output extends PwDbOutput { xml.endTag(null, PwDatabaseV4XML.ElemBinary); } - - private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { - int valLength = (int) value.length(); // TODO verify - if (valLength > 0) { - byte[] buffer = new byte[valLength]; - value.getData().read(buffer, 0, valLength); // TODO Nullable + /* + TODO Make with pipe + private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + try (InputStream inputStream = value.getData()) { + if (inputStream == null) { + Log.e(TAG, "Can't write a null input stream."); + return; + } if (value.isProtected()) { xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); - byte[] encoded = new byte[valLength]; - randomStream.processBytes(buffer, 0, valLength, encoded, 0); - xml.text(String.valueOf(Base64Coder.encode(encoded))); + try (InputStream cypherInputStream = + IOUtil.pipe(inputStream, + o -> new org.spongycastle.crypto.io.CipherOutputStream(o, randomStream))) { + writeInputStreamInBase64(cypherInputStream); + } } else { if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); - byte[] compressData = MemUtil.compress(buffer); - //InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, valLength); // TODO buffer generalize - //gzipInputStream.read(buffer, 0, valLength); - xml.text(String.valueOf(Base64Coder.encode(compressData))); // TODO Compress all - /* - // TODO be a problem not the same length - MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() { - @Override - public void doAction(byte[] buffer) throws IOException { - xml.text(String.valueOf(Base64Coder.encode(buffer))); - } - }); - */ - } - else { - xml.text(String.valueOf(Base64Coder.encode(buffer))); - /* - // TODO Debug - MemUtil.readBytes(value.getData(), - (int) value.length(), - buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer)))); - */ + try (InputStream gZipInputStream = + IOUtil.pipe(inputStream, GZIPOutputStream::new, (int) value.length())) { + writeInputStreamInBase64(gZipInputStream); + } + + } else { + writeInputStreamInBase64(inputStream); } } - - // TODO remove buffer } } - + + private void writeInputStreamInBase64(InputStream inputStream) throws IOException { + try (InputStream base64InputStream = + IOUtil.pipe(inputStream, + o -> new Base64OutputStream(o, DEFAULT))) { + MemUtil.readBytes(base64InputStream, + buffer -> xml.text(Arrays.toString(buffer))); + } + } + //*/ + + //* + private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + + int valLength = (int) value.length(); + if (valLength > 0) { + byte[] buffer = new byte[valLength]; + if (valLength == value.getData().read(buffer, 0, valLength)) { + + if (value.isProtected()) { + xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); + + byte[] encoded = new byte[valLength]; + randomStream.processBytes(buffer, 0, valLength, encoded, 0); + xml.text(String.valueOf(Base64Coder.encode(encoded))); + + } else { + if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); + + byte[] compressData = MemUtil.compress(buffer); + xml.text(String.valueOf(Base64Coder.encode(compressData))); + + } else { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + } + } + } else { + Log.e(TAG, "Unable to read the stream of the protected binary"); + } + } + } + //*/ + private void writeObject(String name, String value, boolean filterXmlChars) throws IllegalArgumentException, IllegalStateException, IOException { assert(name != null && value != null); diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index d783c93ac..42012767c 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -21,12 +21,21 @@ package com.kunzisoft.keepass.utils; import android.os.Parcel; import android.os.Parcelable; +import android.support.annotation.NonNull; +import android.util.Log; + +import com.kunzisoft.keepass.stream.ActionReadBytes; + +import org.apache.commons.io.IOUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -34,8 +43,11 @@ import java.util.zip.GZIPOutputStream; public class MemUtil { + private static final String TAG = MemUtil.class.getName(); + public static final int BUFFER_SIZE_BYTES = 3 * 128; + public static void copyStream(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[1024]; + byte[] buf = new byte[BUFFER_SIZE_BYTES]; int read; try { while ((read = in.read(buf)) != -1) { @@ -46,6 +58,40 @@ public class MemUtil { } } + public static void readBytes(@NonNull InputStream inputStream, ActionReadBytes actionReadBytes) + throws IOException { + byte[] buffer = new byte[MemUtil.BUFFER_SIZE_BYTES]; + int read = 0; + while (read != -1) { + read = inputStream.read(buffer, 0, buffer.length); + if (read != -1) { + byte[] optimizedBuffer; + if (buffer.length == read) { + optimizedBuffer = buffer; + } else { + optimizedBuffer = Arrays.copyOf(buffer, read); + } + actionReadBytes.doAction(optimizedBuffer); + } + } + + /* + byte[] buffer = new byte[BUFFER_SIZE_BYTES]; + // To create the last buffer who is smaller + long numberOfFullBuffer = length / buffer.length; + long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; + int read = 0; + //if (protectedBinary.length() > 0) { + while (read < length) { + // Create the last smaller buffer + if (read >= sizeOfFullBuffers) + buffer = new byte[(int) (length % buffer.length)]; + read += inputStream.read(buffer, 0, buffer.length); + actionReadBytes.doAction(buffer); + } + //*/ + } + public static byte[] decompress(byte[] input) throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(input); GZIPInputStream gzis = new GZIPInputStream(bais); @@ -67,7 +113,40 @@ public class MemUtil { return baos.toByteArray(); } - // For writing to a Parcel + /** + * Compresses the input data using GZip and outputs the compressed data. + * + * @param input + * An {@link InputStream} containing the input raw data. + * + * @return An {@link InputStream} to the compressed data. + */ + public static InputStream compress(final InputStream input) { + final PipedInputStream compressedDataStream = new PipedInputStream(3 * 128); + Log.d(TAG, "About to compress input data using gzip asynchronously..."); + PipedOutputStream compressionOutput; + GZIPOutputStream gzipCompressedDataStream = null; + try { + compressionOutput = new PipedOutputStream(compressedDataStream); + gzipCompressedDataStream = new GZIPOutputStream(compressionOutput); + IOUtils.copy(input, gzipCompressedDataStream); + Log.e(TAG, "Successfully compressed input data using gzip."); + } catch (IOException e) { + Log.e(TAG, "Failed to compress input data.", e); + } finally { + if (gzipCompressedDataStream != null) { + try { + gzipCompressedDataStream.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close gzip output stream.", e); + } + } + } + return compressedDataStream; + } + + + // For writing to a Parcel public static void writeParcelableMap( Parcel parcel, int flags, Map map) { parcel.writeInt(map.size());