From bb3fb26847076851bc870a574b3fc6f723dfe682 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Mon, 3 Sep 2018 12:00:57 +0200 Subject: [PATCH 1/8] Update gradle --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1edafc726..f04d76bcd 100644 --- a/build.gradle +++ b/build.gradle @@ -6,9 +6,10 @@ buildscript { url 'https://maven.google.com/' name 'Google' } + google() } dependencies { - classpath 'com.android.tools.build:gradle:3.1.3' + classpath 'com.android.tools.build:gradle:3.1.4' } } From 9130a3851f19557a73dddca6f2a87d5f0fc248df Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Mon, 3 Sep 2018 20:06:50 +0200 Subject: [PATCH 2/8] Fix Out of memory for large attachment #115 --- .../keepass/database/BinaryPool.java | 26 +++++++----- .../kunzisoft/keepass/database/Database.java | 3 +- .../database/load/ImporterFactory.java | 9 ++-- .../keepass/database/load/ImporterV4.java | 41 +++++++++++-------- .../keepass/database/save/PwDbV4Output.java | 2 +- .../database/security/ProtectedBinary.java | 16 +++++++- .../keepass/stream/LEDataInputStream.java | 28 ++++++++++++- 7 files changed, 91 insertions(+), 34 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java index 84fed9719..5acfd27c9 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java @@ -63,30 +63,36 @@ public class BinaryPool { @Override public boolean operate(PwEntryV4 entry) { for (PwEntryV4 histEntry : entry.getHistory()) { - poolAdd(histEntry.getBinaries()); + add(histEntry.getBinaries()); } - poolAdd(entry.getBinaries()); + add(entry.getBinaries()); return true; } } - private void poolAdd(Map dict) { + private void add(Map dict) { for (ProtectedBinary pb : dict.values()) { - poolAdd(pb); + add(pb); } } - public void poolAdd(ProtectedBinary pb) { - assert(pb != null); + public void add(ProtectedBinary pb) { + assert(pb != null); + if (findKey(pb) != -1) return; - if (poolFind(pb) != -1) return; - - pool.put(pool.size(), pb); + pool.put(findUnusedKey(), pb); } + + public int findUnusedKey() { + int unusedKey = pool.size(); + while(get(unusedKey) != null) + unusedKey++; + return unusedKey; + } - public int poolFind(ProtectedBinary pb) { + public int findKey(ProtectedBinary pb) { for (Entry pair : pool.entrySet()) { if (pair.getValue().equals(pb)) return pair.getKey(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/Database.java b/app/src/main/java/com/kunzisoft/keepass/database/Database.java index 6dd484e49..fc822cc6e 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/Database.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/Database.java @@ -151,7 +151,8 @@ public class Database { // We'll end up reading 8 bytes to identify the header. Might as well use two extra. bis.mark(10); - Importer databaseImporter = ImporterFactory.createImporter(bis, debug); + // Get the file directory to save the attachments + Importer databaseImporter = ImporterFactory.createImporter(bis, ctx.getFilesDir(), debug); bis.reset(); // Return to the start diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java index 385159fd7..02cefcb7e 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java @@ -24,15 +24,16 @@ import com.kunzisoft.keepass.database.PwDbHeaderV4; import com.kunzisoft.keepass.database.exception.InvalidDBSignatureException; import com.kunzisoft.keepass.stream.LEDataInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; public class ImporterFactory { - public static Importer createImporter(InputStream is) throws InvalidDBSignatureException, IOException { - return createImporter(is, false); + public static Importer createImporter(InputStream is, File streamDir) throws InvalidDBSignatureException, IOException { + return createImporter(is, streamDir,false); } - public static Importer createImporter(InputStream is, boolean debug) throws InvalidDBSignatureException, IOException { + public static Importer createImporter(InputStream is, File streamDir, boolean debug) throws InvalidDBSignatureException, IOException { int sig1 = LEDataInputStream.readInt(is); int sig2 = LEDataInputStream.readInt(is); @@ -43,7 +44,7 @@ public class ImporterFactory { return new ImporterV3(); } else if ( PwDbHeaderV4.matchesHeader(sig1, sig2) ) { - return new ImporterV4(); + return new ImporterV4(streamDir); } throw new InvalidDBSignatureException(); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 7719019a4..8d88771c9 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -23,6 +23,7 @@ import com.kunzisoft.keepass.R; import com.kunzisoft.keepass.crypto.CipherFactory; import com.kunzisoft.keepass.crypto.PwStreamCipherFactory; import com.kunzisoft.keepass.crypto.engine.CipherEngine; +import com.kunzisoft.keepass.database.BinaryPool; import com.kunzisoft.keepass.database.ITimeLogger; import com.kunzisoft.keepass.database.PwCompressionAlgorithm; import com.kunzisoft.keepass.database.PwDatabase; @@ -54,6 +55,8 @@ import org.xmlpull.v1.XmlPullParser; import org.xmlpull.v1.XmlPullParserException; import org.xmlpull.v1.XmlPullParserFactory; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -83,9 +86,11 @@ public class ImporterV4 extends Importer { private byte[] pbHeader = null; private long version; Calendar utcCal; + private File streamDir; - public ImporterV4() { - utcCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + public ImporterV4(File streamDir) { + this.utcCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + this.streamDir = streamDir; } @Override @@ -223,7 +228,8 @@ public class ImporterV4 extends Importer { byte[] data = new byte[0]; if (size > 0) { - data = lis.readBytes(size); + if (fieldId != PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary) + data = lis.readBytes(size); } boolean result = true; @@ -238,20 +244,23 @@ public class ImporterV4 extends Importer { header.innerRandomStreamKey = data; break; case PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary: - if (data.length < 1) throw new IOException("Invalid binary format"); - byte flag = data[0]; - boolean prot = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != - PwDbHeaderV4.KdbxBinaryFlags.None; - - byte[] bin = new byte[data.length - 1]; - System.arraycopy(data, 1, bin, 0, data.length-1); - ProtectedBinary pb = new ProtectedBinary(prot, bin); - db.getBinPool().poolAdd(pb); - - if (prot) { - Arrays.fill(data, (byte)0); - } + byte flag = lis.readBytes(1)[0]; + boolean protectedFlag = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != + PwDbHeaderV4.KdbxBinaryFlags.None; + // Read in a file + BinaryPool binaryPool = db.getBinPool(); + int binaryKey = binaryPool.findUnusedKey(); + File file = new File(streamDir, String.valueOf(binaryKey)); + FileOutputStream outputStream = new FileOutputStream(file); + try { + lis.readBytes(size - 1, outputStream::write); + } finally { + outputStream.close(); + } + ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file); + binaryPool.add(protectedBinary); break; + default: assert(false); break; diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 53b4f5701..59f8ac0a3 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -413,7 +413,7 @@ public class PwDbV4Output extends PwDbOutput { xml.startTag(null, PwDatabaseV4XML.ElemValue); String strRef = null; if (allowRef) { - int ref = mPM.getBinPool().poolFind(value); + int ref = mPM.getBinPool().findKey(value); strRef = Integer.toString(ref); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 463b7b604..3b6e6aed6 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -22,6 +22,7 @@ package com.kunzisoft.keepass.database.security; import android.os.Parcel; import android.os.Parcelable; +import java.io.File; import java.util.Arrays; public class ProtectedBinary implements Parcelable { @@ -30,12 +31,14 @@ public class ProtectedBinary implements Parcelable { private boolean protect; private byte[] data; + private File dataFile; public boolean isProtected() { return protect; } public int length() { + // TODO File length if (data == null) { return 0; } @@ -49,11 +52,19 @@ public class ProtectedBinary implements Parcelable { public ProtectedBinary(boolean enableProtection, byte[] data) { this.protect = enableProtection; this.data = data; + this.dataFile = null; } + public ProtectedBinary(boolean enableProtection, File dataFile) { + this.protect = enableProtection; + this.data = new byte[0]; + this.dataFile = dataFile; + } + public ProtectedBinary(Parcel in) { protect = in.readByte() != 0; in.readByteArray(data); + dataFile = new File(in.readString()); } // TODO: replace the byte[] with something like ByteBuffer to make the return @@ -63,7 +74,9 @@ public class ProtectedBinary implements Parcelable { } public boolean equals(ProtectedBinary rhs) { - return (protect == rhs.protect) && Arrays.equals(data, rhs.data); + return (protect == rhs.protect) + && Arrays.equals(data, rhs.data) + && dataFile.equals(rhs.dataFile); } @Override @@ -75,6 +88,7 @@ public class ProtectedBinary implements Parcelable { public void writeToParcel(Parcel dest, int flags) { dest.writeByte((byte) (protect ? 1 : 0)); dest.writeByteArray(data); + dest.writeString(dataFile.getAbsolutePath()); } public static final Creator CREATOR = new Creator() { diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java index a213f7c76..dab42aca6 100644 --- a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java +++ b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java @@ -106,8 +106,9 @@ public class LEDataInputStream extends InputStream { } public byte[] readBytes(int length) throws IOException { + // TODO Exception max length < buffer size byte[] buf = new byte[length]; - + int count = 0; while ( count < length ) { int read = read(buf, count, length - count); @@ -126,6 +127,31 @@ public class LEDataInputStream extends InputStream { return buf; } + public void readBytes(int length, ActionReadBytes actionReadBytes) throws IOException { + byte[] buffer = new byte[1024]; + + int offset = 0; + int read = 0; + while ( offset < length && read != -1) { + + int tempLength = buffer.length; + // If buffer not needed + if (length - offset < tempLength) + tempLength = length - offset; + read = read(buffer, 0, tempLength); + actionReadBytes.doAction(buffer); + offset += read; + } + } + + public interface ActionReadBytes { + /** + * Called after each buffer fill + * @param buffer filled + */ + void doAction(byte[] buffer) throws IOException; + } + public static int readUShort(InputStream is) throws IOException { byte[] buf = new byte[2]; From d0560677fa5f49c666e097b7a64142547530e26b Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Wed, 5 Sep 2018 22:02:24 +0200 Subject: [PATCH 3/8] Temp commit --- app/build.gradle | 2 + .../kunzisoft/keepass/database/PwEntryV4.java | 3 +- .../keepass/database/load/ImporterV4.java | 18 ++--- .../save/PwDbInnerHeaderOutputV4.java | 31 ++++++-- .../keepass/database/save/PwDbV4Output.java | 45 ++++++++---- .../database/security/ProtectedBinary.java | 71 +++++++++++++------ .../keepass/stream/ActionReadBytes.java | 11 +++ .../keepass/stream/LEDataInputStream.java | 17 ++--- .../com/kunzisoft/keepass/utils/MemUtil.java | 29 +++++--- 9 files changed, 152 insertions(+), 75 deletions(-) create mode 100644 app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java diff --git a/app/build.gradle b/app/build.gradle index e53c02032..48b1234e7 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -105,6 +105,8 @@ dependencies { implementation 'commons-collections:commons-collections:3.2.1' // Base64 implementation 'biz.source_code:base64coder:2010-12-19' + // IO-Extras + implementation 'com.github.davidmoten:io-extras:0.1' implementation 'com.google.code.gson:gson:2.8.4' implementation 'com.google.guava:guava:23.0-android' // Icon pack, classic for all, material for libre and pro diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java b/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java index 921a44141..677db7f26 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java @@ -93,8 +93,7 @@ public class PwEntryV4 extends PwEntry implements ITimeLogger { parentGroupLastMod = in.readParcelable(PwDate.class.getClassLoader()); customData = MemUtil.readStringParcelableMap(in); fields = in.readParcelable(ExtraFields.class.getClassLoader()); - // TODO binaries takes too much memory for parcelable - // binaries = MemUtil.readStringParcelableMap(in, ProtectedBinary.class); + binaries = MemUtil.readStringParcelableMap(in, ProtectedBinary.class); foregroundColor = in.readString(); backgroupColor = in.readString(); overrideURL = in.readString(); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 8d88771c9..7dda69710 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -251,13 +251,10 @@ public class ImporterV4 extends Importer { BinaryPool binaryPool = db.getBinPool(); int binaryKey = binaryPool.findUnusedKey(); File file = new File(streamDir, String.valueOf(binaryKey)); - FileOutputStream outputStream = new FileOutputStream(file); - try { - lis.readBytes(size - 1, outputStream::write); - } finally { - outputStream.close(); - } - ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file); + try (FileOutputStream outputStream = new FileOutputStream(file)) { + lis.readBytes(size - 1, outputStream::write); + } + ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, size -1); binaryPool.add(protectedBinary); break; @@ -1077,10 +1074,12 @@ public class ImporterV4 extends Importer { byte[] buf = ProcessNode(xpp); - if ( buf != null ) return new ProtectedBinary(true, buf); + if ( buf != null ) + return new ProtectedBinary(true, buf); String base64 = ReadString(xpp); - if ( base64.length() == 0 ) return ProtectedBinary.EMPTY; + if ( base64.length() == 0 ) + return ProtectedBinary.EMPTY; byte[] data = Base64Coder.decode(base64); @@ -1121,6 +1120,7 @@ public class ImporterV4 extends Importer { if ( xpp.getAttributeCount() > 0 ) { String protect = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrProtected); if ( protect != null && protect.equalsIgnoreCase(PwDatabaseV4XML.ValTrue) ) { + // TODO stream for encrypted data String encrypted = ReadStringRaw(xpp); if ( encrypted.length() > 0 ) { diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java index 89dc3145e..b54eb6d92 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java @@ -25,8 +25,8 @@ import com.kunzisoft.keepass.database.security.ProtectedBinary; import com.kunzisoft.keepass.stream.LEDataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; -import java.util.Arrays; public class PwDbInnerHeaderOutputV4 { private PwDatabaseV4 db; @@ -50,19 +50,36 @@ public class PwDbInnerHeaderOutputV4 { los.writeInt(streamKeySize); los.write(header.innerRandomStreamKey); - for (ProtectedBinary bin : db.getBinPool().binaries()) { + for (ProtectedBinary protectedBinary : db.getBinPool().binaries()) { byte flag = PwDbHeaderV4.KdbxBinaryFlags.None; - if (bin.isProtected()) { + if (protectedBinary.isProtected()) { flag |= PwDbHeaderV4.KdbxBinaryFlags.Protected; } - byte[] binData = bin.getData(); los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary); - los.writeInt(bin.length() + 1); + los.writeInt((int) protectedBinary.length() + 1); // TODO verify los.write(flag); - los.write(binData); - Arrays.fill(binData, (byte)0); + /* + MemUtil.readBytes(protectedBinary.getData(), + (int) protectedBinary.length(), + buffer -> los.write(buffer)); + */ + + byte[] buffer = new byte[3 * 256]; // TODO buffer generalize + InputStream fileInputStream = protectedBinary.getData(); // TODO Nullable + // To create the last buffer who is smaller + long numberOfFullBuffer = protectedBinary.length() / buffer.length; + long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; + int read = 0; + //if (protectedBinary.length() > 0) { + while (read < protectedBinary.length()) { + // Create the last smaller buffer + if (read >= sizeOfFullBuffers) + buffer = new byte[(int) (protectedBinary.length() % buffer.length)]; + read += fileInputStream.read(buffer, 0, buffer.length); + los.write(buffer); + } } los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.EndOfHeader); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 59f8ac0a3..04b207939 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -54,11 +54,13 @@ import com.kunzisoft.keepass.utils.EmptyUtils; import com.kunzisoft.keepass.utils.MemUtil; import com.kunzisoft.keepass.utils.Types; +import org.davidmoten.io.extras.IOUtil; import org.joda.time.DateTime; import org.spongycastle.crypto.StreamCipher; import org.xmlpull.v1.XmlSerializer; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -429,26 +431,43 @@ public class PwDbV4Output extends PwDbOutput { } private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + int valLength = (int) value.length(); // TODO verify + if (value.isProtected()) { xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); - - int valLength = value.length(); - if (valLength > 0) { - byte[] encoded = new byte[valLength]; - randomStream.processBytes(value.getData(), 0, valLength, encoded, 0); - - xml.text(String.valueOf(Base64Coder.encode(encoded))); - } + + byte[] buffer = new byte[valLength]; // TODO buffer generalize + if (valLength > 0) { + byte[] encoded = new byte[valLength]; + value.getData().read(buffer, 0, buffer.length); // TODO throw exception if length != read // NUllable + randomStream.processBytes(buffer, 0, valLength, encoded, 0); + xml.text(String.valueOf(Base64Coder.encode(encoded))); + } } else { if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); - byte[] raw = value.getData(); - byte[] compressed = MemUtil.compress(raw); - xml.text(String.valueOf(Base64Coder.encode(compressed))); + + InputStream fileInputStream = value.getData(); // TODO Nullable + InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, 1024); // TODO buffer generalize + byte[] buffer = new byte[valLength]; // TODO buffer generalize + if (valLength > 0) { + int read = gzipInputStream.read(buffer, 0, buffer.length); + xml.text(String.valueOf(Base64Coder.encode(buffer))); // TODO Compress all + } + /* + // TODO be a problem not the same length + MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() { + @Override + public void doAction(byte[] buffer) throws IOException { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + } + }); + */ } else { - byte[] raw = value.getData(); - xml.text(String.valueOf(Base64Coder.encode(raw))); + MemUtil.readBytes(value.getData(), + (int) value.length(), + buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer)))); } } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 3b6e6aed6..5263b3d3f 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -22,8 +22,13 @@ package com.kunzisoft.keepass.database.security; import android.os.Parcel; import android.os.Parcelable; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; +import java.util.Objects; public class ProtectedBinary implements Parcelable { @@ -32,54 +37,73 @@ public class ProtectedBinary implements Parcelable { private boolean protect; private byte[] data; private File dataFile; - + private int size; + public boolean isProtected() { return protect; } - public int length() { - // TODO File length - if (data == null) { - return 0; - } - return data.length; + public long length() { + return size; } public ProtectedBinary() { - this(false, new byte[0]); + this.protect = false; + this.data = null; + this.dataFile = null; + this.size = 0; } public ProtectedBinary(boolean enableProtection, byte[] data) { this.protect = enableProtection; this.data = data; this.dataFile = null; + this.size = data.length; } - public ProtectedBinary(boolean enableProtection, File dataFile) { + public ProtectedBinary(boolean enableProtection, File dataFile, int size) { this.protect = enableProtection; - this.data = new byte[0]; + this.data = null; this.dataFile = dataFile; + this.size = size; } public ProtectedBinary(Parcel in) { protect = in.readByte() != 0; in.readByteArray(data); dataFile = new File(in.readString()); - } - - // TODO: replace the byte[] with something like ByteBuffer to make the return - // value immutable, so we don't have to worry about making deep copies - public byte[] getData() { - return data; - } - - public boolean equals(ProtectedBinary rhs) { - return (protect == rhs.protect) - && Arrays.equals(data, rhs.data) - && dataFile.equals(rhs.dataFile); + size = in.readInt(); } - @Override + public InputStream getData() throws IOException { + if (data != null) + return new ByteArrayInputStream(data); + else if (dataFile != null) + return new FileInputStream(dataFile); + else + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ProtectedBinary that = (ProtectedBinary) o; + return protect == that.protect && + size == that.size && + Arrays.equals(data, that.data) && + Objects.equals(dataFile, that.dataFile); + } + + @Override + public int hashCode() { + + int result = Objects.hash(protect, dataFile, size); + result = 31 * result + Arrays.hashCode(data); + return result; + } + + @Override public int describeContents() { return 0; } @@ -89,6 +113,7 @@ public class ProtectedBinary implements Parcelable { dest.writeByte((byte) (protect ? 1 : 0)); dest.writeByteArray(data); dest.writeString(dataFile.getAbsolutePath()); + dest.writeInt(size); } public static final Creator CREATOR = new Creator() { diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java b/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java new file mode 100644 index 000000000..d72ecb2a4 --- /dev/null +++ b/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java @@ -0,0 +1,11 @@ +package com.kunzisoft.keepass.stream; + +import java.io.IOException; + +public interface ActionReadBytes { + /** + * Called after each buffer fill + * @param buffer filled + */ + void doAction(byte[] buffer) throws IOException; +} diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java index dab42aca6..686dac698 100644 --- a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java +++ b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java @@ -23,6 +23,7 @@ import com.kunzisoft.keepass.utils.Types; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.UUID; @@ -128,30 +129,26 @@ public class LEDataInputStream extends InputStream { } public void readBytes(int length, ActionReadBytes actionReadBytes) throws IOException { - byte[] buffer = new byte[1024]; + //MemUtil.readBytes(baseStream, length, actionReadBytes); + byte[] buffer = new byte[3 * 256]; int offset = 0; int read = 0; while ( offset < length && read != -1) { - int tempLength = buffer.length; // If buffer not needed if (length - offset < tempLength) tempLength = length - offset; read = read(buffer, 0, tempLength); + if (read >= 0 && buffer.length != read) { // TODO Better perf + byte[] tmpBytes = buffer; + buffer = Arrays.copyOf(tmpBytes, read); + } actionReadBytes.doAction(buffer); offset += read; } } - public interface ActionReadBytes { - /** - * Called after each buffer fill - * @param buffer filled - */ - void doAction(byte[] buffer) throws IOException; - } - public static int readUShort(InputStream is) throws IOException { byte[] buf = new byte[2]; diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index 92e516e86..a663a1c71 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -22,13 +22,15 @@ package com.kunzisoft.keepass.utils; import android.os.Parcel; import android.os.Parcelable; +import com.kunzisoft.keepass.stream.ActionReadBytes; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; public class MemUtil { public static byte[] decompress(byte[] input) throws IOException { @@ -40,16 +42,21 @@ public class MemUtil { return baos.toByteArray(); } - - public static byte[] compress(byte[] input) throws IOException { - ByteArrayInputStream bais = new ByteArrayInputStream(input); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - GZIPOutputStream gzos = new GZIPOutputStream(baos); - Util.copyStream(bais, gzos); - gzos.close(); - - return baos.toByteArray(); + + public static void readBytes(InputStream inputStream, int length, ActionReadBytes action) throws IOException { + byte[] buffer = new byte[3 * 256]; // TODO buffer generalize + // To create the last buffer who is smaller + long numberOfFullBuffer = length / buffer.length; + long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; + int read = 0; + //if (protectedBinary.length() > 0) { + while (read < length) { + // Create the last smaller buffer + if (read >= sizeOfFullBuffers) + buffer = new byte[(length % buffer.length)]; + read += inputStream.read(buffer, 0, buffer.length); + action.doAction(buffer); + } } // For writing to a Parcel From df80084822564ffb7d1233330c04c211f5190592 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Thu, 6 Sep 2018 22:05:31 +0200 Subject: [PATCH 4/8] Fix attachment recovery --- .../keepass/database/save/PwDbV4Output.java | 72 ++++++++++--------- .../keepass/stream/LEDataInputStream.java | 28 +++++--- .../com/kunzisoft/keepass/utils/MemUtil.java | 29 +++----- 3 files changed, 65 insertions(+), 64 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 04b207939..2b4a85657 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -54,13 +54,11 @@ import com.kunzisoft.keepass.utils.EmptyUtils; import com.kunzisoft.keepass.utils.MemUtil; import com.kunzisoft.keepass.utils.Types; -import org.davidmoten.io.extras.IOUtil; import org.joda.time.DateTime; import org.spongycastle.crypto.StreamCipher; import org.xmlpull.v1.XmlSerializer; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -431,46 +429,50 @@ public class PwDbV4Output extends PwDbOutput { } private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { - int valLength = (int) value.length(); // TODO verify + int valLength = (int) value.length(); // TODO verify + if (valLength > 0) { + byte[] buffer = new byte[valLength]; - if (value.isProtected()) { - xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); + value.getData().read(buffer, 0, valLength); // TODO Nullable + + if (value.isProtected()) { + xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); - byte[] buffer = new byte[valLength]; // TODO buffer generalize - if (valLength > 0) { byte[] encoded = new byte[valLength]; - value.getData().read(buffer, 0, buffer.length); // TODO throw exception if length != read // NUllable randomStream.processBytes(buffer, 0, valLength, encoded, 0); xml.text(String.valueOf(Base64Coder.encode(encoded))); - } - - } else { - if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { - xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); - InputStream fileInputStream = value.getData(); // TODO Nullable - InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, 1024); // TODO buffer generalize - byte[] buffer = new byte[valLength]; // TODO buffer generalize - if (valLength > 0) { - int read = gzipInputStream.read(buffer, 0, buffer.length); - xml.text(String.valueOf(Base64Coder.encode(buffer))); // TODO Compress all + } else { + if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); + + byte[] compressData = MemUtil.compress(buffer); + //InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, valLength); // TODO buffer generalize + //gzipInputStream.read(buffer, 0, valLength); + xml.text(String.valueOf(Base64Coder.encode(compressData))); // TODO Compress all + /* + // TODO be a problem not the same length + MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() { + @Override + public void doAction(byte[] buffer) throws IOException { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + } + }); + */ } - /* - // TODO be a problem not the same length - MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() { - @Override - public void doAction(byte[] buffer) throws IOException { - xml.text(String.valueOf(Base64Coder.encode(buffer))); - } - }); - */ - } else { - MemUtil.readBytes(value.getData(), - (int) value.length(), - buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer)))); - } - - } + else { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + /* + // TODO Debug + MemUtil.readBytes(value.getData(), + (int) value.length(), + buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer)))); + */ + } + } + + // TODO remove buffer + } } private void writeObject(String name, String value, boolean filterXmlChars) throws IllegalArgumentException, IllegalStateException, IOException { diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java index 686dac698..5d4cbfb88 100644 --- a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java +++ b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java @@ -129,22 +129,28 @@ public class LEDataInputStream extends InputStream { } public void readBytes(int length, ActionReadBytes actionReadBytes) throws IOException { - //MemUtil.readBytes(baseStream, length, actionReadBytes); - byte[] buffer = new byte[3 * 256]; + int bufferSize = 256; + byte[] buffer = new byte[bufferSize]; int offset = 0; int read = 0; while ( offset < length && read != -1) { - int tempLength = buffer.length; - // If buffer not needed - if (length - offset < tempLength) - tempLength = length - offset; - read = read(buffer, 0, tempLength); - if (read >= 0 && buffer.length != read) { // TODO Better perf - byte[] tmpBytes = buffer; - buffer = Arrays.copyOf(tmpBytes, read); + + // To reduce the buffer for the last bytes reads + if (length - offset < bufferSize) { + bufferSize = length - offset; + buffer = new byte[bufferSize]; } - actionReadBytes.doAction(buffer); + read = read(buffer, 0, bufferSize); + + // To get only the bytes read + byte[] optimizedBuffer; + if (read >= 0 && buffer.length > read) { + optimizedBuffer = Arrays.copyOf(buffer, read); + } else { + optimizedBuffer = buffer; + } + actionReadBytes.doAction(optimizedBuffer); offset += read; } } diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index a663a1c71..70004d304 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -22,15 +22,13 @@ package com.kunzisoft.keepass.utils; import android.os.Parcel; import android.os.Parcelable; -import com.kunzisoft.keepass.stream.ActionReadBytes; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; public class MemUtil { public static byte[] decompress(byte[] input) throws IOException { @@ -43,21 +41,16 @@ public class MemUtil { return baos.toByteArray(); } - public static void readBytes(InputStream inputStream, int length, ActionReadBytes action) throws IOException { - byte[] buffer = new byte[3 * 256]; // TODO buffer generalize - // To create the last buffer who is smaller - long numberOfFullBuffer = length / buffer.length; - long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; - int read = 0; - //if (protectedBinary.length() > 0) { - while (read < length) { - // Create the last smaller buffer - if (read >= sizeOfFullBuffers) - buffer = new byte[(length % buffer.length)]; - read += inputStream.read(buffer, 0, buffer.length); - action.doAction(buffer); - } - } + public static byte[] compress(byte[] input) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(input); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream gzos = new GZIPOutputStream(baos); + Util.copyStream(bais, gzos); + gzos.close(); + + return baos.toByteArray(); + } // For writing to a Parcel public static void writeParcelableMap( From 9ff6e7a0800cf36dc4332a2c068b251bd6d01701 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Sat, 15 Sep 2018 17:22:42 +0200 Subject: [PATCH 5/8] Clear the cache to keep a small memory when the app is down --- app/build.gradle | 1 + .../java/com/kunzisoft/keepass/app/App.java | 2 +- .../keepass/database/BinaryPool.java | 2 + .../kunzisoft/keepass/database/Database.java | 51 +++++++++++-------- .../keepass/database/PwDatabase.java | 2 + .../keepass/database/PwDatabaseV4.java | 5 ++ .../database/security/ProtectedBinary.java | 14 +++-- .../keepass/password/PasswordActivity.java | 4 +- 8 files changed, 55 insertions(+), 26 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 48b1234e7..67c9602ea 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -103,6 +103,7 @@ dependencies { annotationProcessor "com.github.hotchemi:permissionsdispatcher-processor:$permissionDispatcherVersion" // Apache Commons Collections implementation 'commons-collections:commons-collections:3.2.1' + implementation 'org.apache.commons:commons-io:1.3.2' // Base64 implementation 'biz.source_code:base64coder:2010-12-19' // IO-Extras diff --git a/app/src/main/java/com/kunzisoft/keepass/app/App.java b/app/src/main/java/com/kunzisoft/keepass/app/App.java index 6a861dcde..346fdd1e4 100644 --- a/app/src/main/java/com/kunzisoft/keepass/app/App.java +++ b/app/src/main/java/com/kunzisoft/keepass/app/App.java @@ -92,7 +92,7 @@ public class App extends MultiDexApplication { @Override public void onTerminate() { if ( db != null ) { - db.clear(); + db.clear(getApplicationContext()); } super.onTerminate(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java index 5acfd27c9..c256d5fd9 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java @@ -51,6 +51,8 @@ public class BinaryPool { } public void clear() { + for (Entry entry: pool.entrySet()) + entry.getValue().clear(); pool.clear(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/Database.java b/app/src/main/java/com/kunzisoft/keepass/database/Database.java index fc822cc6e..ce5c4ce50 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/Database.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/Database.java @@ -39,6 +39,8 @@ import com.kunzisoft.keepass.icons.IconDrawableFactory; import com.kunzisoft.keepass.tasks.ProgressTaskUpdater; import com.kunzisoft.keepass.utils.UriUtil; +import org.apache.commons.io.FileUtils; + import java.io.BufferedInputStream; import java.io.File; import java.io.FileNotFoundException; @@ -58,7 +60,7 @@ public class Database { private static final String TAG = Database.class.getName(); - private PwDatabase pm; + private PwDatabase pwDatabase; private Uri mUri; private SearchDbHelper searchHelper; private boolean readOnly = false; @@ -69,11 +71,11 @@ public class Database { private boolean loaded = false; public PwDatabase getPwDatabase() { - return pm; + return pwDatabase; } public void setPwDatabase(PwDatabase pm) { - this.pm = pm; + this.pwDatabase = pm; } public void setUri(Uri mUri) { @@ -156,20 +158,20 @@ public class Database { bis.reset(); // Return to the start - pm = databaseImporter.openDatabase(bis, password, keyFileInputStream, progressTaskUpdater); - if ( pm != null ) { + pwDatabase = databaseImporter.openDatabase(bis, password, keyFileInputStream, progressTaskUpdater); + if ( pwDatabase != null ) { try { - switch (pm.getVersion()) { + switch (pwDatabase.getVersion()) { case V3: - PwGroupV3 rootV3 = ((PwDatabaseV3) pm).getRootGroup(); - ((PwDatabaseV3) pm).populateGlobals(rootV3); - passwordEncodingError = !pm.validatePasswordEncoding(password); + PwGroupV3 rootV3 = ((PwDatabaseV3) pwDatabase).getRootGroup(); + ((PwDatabaseV3) pwDatabase).populateGlobals(rootV3); + passwordEncodingError = !pwDatabase.validatePasswordEncoding(password); searchHelper = new SearchDbHelper.SearchDbHelperV3(ctx); break; case V4: - PwGroupV4 rootV4 = ((PwDatabaseV4) pm).getRootGroup(); - ((PwDatabaseV4) pm).populateGlobals(rootV4); - passwordEncodingError = !pm.validatePasswordEncoding(password); + PwGroupV4 rootV4 = ((PwDatabaseV4) pwDatabase).getRootGroup(); + ((PwDatabaseV4) pwDatabase).populateGlobals(rootV4); + passwordEncodingError = !pwDatabase.validatePasswordEncoding(password); searchHelper = new SearchDbHelper.SearchDbHelperV4(ctx); break; } @@ -188,11 +190,11 @@ public class Database { public PwGroup search(String str, int max) { if (searchHelper == null) { return null; } try { - switch (pm.getVersion()) { + switch (pwDatabase.getVersion()) { case V3: - return ((SearchDbHelper.SearchDbHelperV3) searchHelper).search(((PwDatabaseV3) pm), str, max); + return ((SearchDbHelper.SearchDbHelperV3) searchHelper).search(((PwDatabaseV3) pwDatabase), str, max); case V4: - return ((SearchDbHelper.SearchDbHelperV4) searchHelper).search(((PwDatabaseV4) pm), str, max); + return ((SearchDbHelper.SearchDbHelperV4) searchHelper).search(((PwDatabaseV4) pwDatabase), str, max); } } catch (Exception e) { Log.e(TAG, "Search can't be performed with this SearchHelper", e); @@ -262,7 +264,7 @@ public class Database { FileOutputStream fos = null; try { fos = new FileOutputStream(tempFile); - PwDbOutput pmo = PwDbOutput.getInstance(pm, fos); + PwDbOutput pmo = PwDbOutput.getInstance(pwDatabase, fos); if (pmo != null) pmo.output(); } catch (Exception e) { @@ -290,7 +292,7 @@ public class Database { OutputStream os = null; try { os = ctx.getContentResolver().openOutputStream(uri); - PwDbOutput pmo = PwDbOutput.getInstance(pm, os); + PwDbOutput pmo = PwDbOutput.getInstance(pwDatabase, os); if (pmo != null) pmo.output(); } catch (Exception e) { @@ -305,10 +307,19 @@ public class Database { } // TODO Clear database when lock broadcast is receive in backstage - public void clear() { + public void clear(Context context) { drawFactory.clearCache(); + // Delete the cache of the database if present + if (pwDatabase != null) + pwDatabase.clearCache(); + // In all cases, delete all the files in the temp dir + try { + FileUtils.cleanDirectory(context.getFilesDir()); + } catch (IOException e) { + Log.e(TAG, "Unable to clear the directory cache.", e); + } - pm = null; + pwDatabase = null; mUri = null; loaded = false; passwordEncodingError = false; @@ -546,7 +557,7 @@ public class Database { case V4: newPwGroup = new PwGroupV4((PwGroupV4) parent); } - newPwGroup.setId(pm.newGroupId()); + newPwGroup.setId(pwDatabase.newGroupId()); } catch (Exception e) { Log.e(TAG, "This version of PwGroup can't be created", e); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java index cd2a797ae..3dda73adf 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java @@ -430,4 +430,6 @@ public abstract class PwDatabase */ public abstract void initNew(String dbPath); + public void clearCache() {} + } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java index 512a8e644..a20ad5a0a 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java @@ -754,4 +754,9 @@ public class PwDatabaseV4 extends PwDatabase { return filename.substring(0, lastExtDot); } + @Override + public void clearCache() { + super.clearCache(); + binPool.clear(); + } } \ No newline at end of file diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 5263b3d3f..355af8eff 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -21,6 +21,7 @@ package com.kunzisoft.keepass.database.security; import android.os.Parcel; import android.os.Parcelable; +import android.util.Log; import java.io.ByteArrayInputStream; import java.io.File; @@ -31,7 +32,8 @@ import java.util.Arrays; import java.util.Objects; public class ProtectedBinary implements Parcelable { - + + private static final String TAG = ProtectedBinary.class.getName(); public final static ProtectedBinary EMPTY = new ProtectedBinary(); private boolean protect; @@ -84,6 +86,12 @@ public class ProtectedBinary implements Parcelable { return null; } + public void clear() { + data = null; + if (dataFile != null && !dataFile.delete()) + Log.e(TAG, "Unable to delete temp file " + dataFile.getAbsolutePath()); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -92,13 +100,13 @@ public class ProtectedBinary implements Parcelable { return protect == that.protect && size == that.size && Arrays.equals(data, that.data) && - Objects.equals(dataFile, that.dataFile); + Objects.equals(dataFile, that.dataFile); // TODO old equals } @Override public int hashCode() { - int result = Objects.hash(protect, dataFile, size); + int result = Objects.hash(protect, dataFile, size); // TODO old hash result = 31 * result + Arrays.hashCode(data); return result; } diff --git a/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java b/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java index 1db624afb..71c246737 100644 --- a/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java +++ b/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java @@ -252,7 +252,7 @@ public class PasswordActivity extends StylishActivity case LockingActivity.RESULT_EXIT_LOCK: case Activity.RESULT_CANCELED: setEmptyViews(); - App.getDB().clear(); + App.getDB().clear(getApplicationContext()); break; } } @@ -938,7 +938,7 @@ public class PasswordActivity extends StylishActivity private void loadDatabase(String password, Uri keyfile) { // Clear before we load Database database = App.getDB(); - database.clear(); + database.clear(getApplicationContext()); // Clear the shutdown flag App.clearShutdown(); From cee7419208c45e721a1112ec0939acc7af3977e8 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Sat, 15 Sep 2018 17:36:14 +0200 Subject: [PATCH 6/8] Fix equals and hashcode for ProtectedBinary --- .../keepass/database/security/ProtectedBinary.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 355af8eff..51c6bc961 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -29,7 +29,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.Objects; public class ProtectedBinary implements Parcelable { @@ -100,13 +99,17 @@ public class ProtectedBinary implements Parcelable { return protect == that.protect && size == that.size && Arrays.equals(data, that.data) && - Objects.equals(dataFile, that.dataFile); // TODO old equals + dataFile != null && + dataFile.equals(that.dataFile); } @Override public int hashCode() { - int result = Objects.hash(protect, dataFile, size); // TODO old hash + int result = 0; + result = 31 * result + (protect ? 1 : 0); + result = 31 * result + dataFile.hashCode(); + result = 31 * result + Integer.valueOf(size).hashCode(); result = 31 * result + Arrays.hashCode(data); return result; } From b96436c906625455b7a6e37ee4dce00affbaafc4 Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Sun, 16 Sep 2018 11:40:22 +0200 Subject: [PATCH 7/8] Change small code --- .../keepass/database/PwDatabase.java | 6 +- .../keepass/database/PwDatabaseV3.java | 3 + .../keepass/database/PwDatabaseV4.java | 1 - .../keepass/database/load/ImporterV4.java | 79 ++++++------------- .../keepass/database/save/PwDbV4Output.java | 15 ++-- .../database/security/ProtectedBinary.java | 17 ++-- .../keepass/stream/LEDataInputStream.java | 19 ++--- .../com/kunzisoft/keepass/utils/MemUtil.java | 19 ++++- .../com/kunzisoft/keepass/utils/Util.java | 19 +---- 9 files changed, 72 insertions(+), 106 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java index 3dda73adf..09b324178 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java @@ -21,7 +21,7 @@ package com.kunzisoft.keepass.database; import com.kunzisoft.keepass.database.exception.InvalidKeyFileException; import com.kunzisoft.keepass.database.exception.KeyFileEmptyException; -import com.kunzisoft.keepass.utils.Util; +import com.kunzisoft.keepass.utils.MemUtil; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -128,7 +128,7 @@ public abstract class PwDatabase assert(keyInputStream != null); ByteArrayOutputStream bos = new ByteArrayOutputStream(); - Util.copyStream(keyInputStream, bos); + MemUtil.copyStream(keyInputStream, bos); byte[] keyData = bos.toByteArray(); ByteArrayInputStream bis = new ByteArrayInputStream(keyData); @@ -430,6 +430,6 @@ public abstract class PwDatabase */ public abstract void initNew(String dbPath); - public void clearCache() {} + public abstract void clearCache(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java index 3c813614e..8be18ff21 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java @@ -376,4 +376,7 @@ public class PwDatabaseV3 extends PwDatabase { return !(omitBackup && isBackup(group)); } + + @Override + public void clearCache() {} } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java index a20ad5a0a..bf7490bf6 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java @@ -756,7 +756,6 @@ public class PwDatabaseV4 extends PwDatabase { @Override public void clearCache() { - super.clearCache(); binPool.clear(); } } \ No newline at end of file diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 7dda69710..221aece5d 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -65,10 +65,8 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.text.ParseException; import java.util.Arrays; -import java.util.Calendar; import java.util.Date; import java.util.Stack; -import java.util.TimeZone; import java.util.UUID; import java.util.zip.GZIPInputStream; @@ -83,13 +81,10 @@ public class ImporterV4 extends Importer { private PwDatabaseV4 db; private byte[] hashOfHeader = null; - private byte[] pbHeader = null; private long version; - Calendar utcCal; private File streamDir; public ImporterV4(File streamDir) { - this.utcCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); this.streamDir = streamDir; } @@ -116,7 +111,7 @@ public class ImporterV4 extends Importer { version = header.getVersion(); hashOfHeader = hh.hash; - pbHeader = hh.header; + byte[] pbHeader = hh.header; db.retrieveMasterKey(password, keyInputStream); db.makeFinalKey(header.masterSeed); @@ -190,7 +185,6 @@ public class ImporterV4 extends Importer { } if ( header.innerRandomStreamKey == null ) { - assert(false); throw new IOException("Invalid stream key."); } @@ -259,7 +253,6 @@ public class ImporterV4 extends Importer { break; default: - assert(false); break; } @@ -354,20 +347,17 @@ public class ImporterV4 extends Importer { } switch ( xpp.getEventType() ) { - case XmlPullParser.START_TAG: - ctx = ReadXmlElement(ctx, xpp); - break; - - case XmlPullParser.END_TAG: - ctx = EndXmlElement(ctx, xpp); - break; + case XmlPullParser.START_TAG: + ctx = ReadXmlElement(ctx, xpp); + break; - default: - assert(false); - break; - + case XmlPullParser.END_TAG: + ctx = EndXmlElement(ctx, xpp); + break; + + default: + break; } - } // Error checks @@ -539,8 +529,8 @@ public class ImporterV4 extends Importer { case Root: if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemGroup) ) { - assert(ctxGroups.size() == 0); - if ( ctxGroups.size() != 0 ) throw new IOException("Group list should be empty."); + if ( ctxGroups.size() != 0 ) + throw new IOException("Group list should be empty."); db.setRootGroup(new PwGroupV4()); ctxGroups.push(db.getRootGroup()); @@ -639,8 +629,6 @@ public class ImporterV4 extends Importer { } else if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemCustomData)) { return SwitchContext(ctx, KdbContext.EntryCustomData, xpp); } else if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemHistory) ) { - assert(!entryInHistory); - if ( ! entryInHistory ) { ctxHistoryBase = ctxEntry; return SwitchContext(ctx, KdbContext.EntryHistory, xpp); @@ -780,8 +768,8 @@ public class ImporterV4 extends Importer { } private KdbContext EndXmlElement(KdbContext ctx, XmlPullParser xpp) throws XmlPullParserException { - assert(xpp.getEventType() == XmlPullParser.END_TAG); - + // (xpp.getEventType() == XmlPullParser.END_TAG); + String name = xpp.getName(); if ( ctx == KdbContext.KeePassFile && name.equalsIgnoreCase(PwDatabaseV4XML.ElemDocNode) ) { return KdbContext.Null; @@ -798,7 +786,7 @@ public class ImporterV4 extends Importer { PwIconCustom icon = new PwIconCustom(customIconID, customIconData); db.addCustomIcon(icon); db.getIconFactory().put(icon); - } else assert(false); + } customIconID = PwDatabase.UUID_ZERO; customIconData = null; @@ -811,7 +799,7 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.CustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem) ) { if ( customDataKey != null && customDataValue != null) { db.putCustomData(customDataKey, customDataValue); - } else assert(false); + } customDataKey = null; customDataValue = null; @@ -838,8 +826,6 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.GroupCustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem)) { if (groupCustomDataKey != null && groupCustomDataValue != null) { ctxGroup.putCustomData(groupCustomDataKey, groupCustomDataKey); - } else { - assert(false); } groupCustomDataKey = null; @@ -885,8 +871,6 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.EntryCustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem)) { if (entryCustomDataKey != null && entryCustomDataValue != null) { ctxEntry.putCustomData(entryCustomDataKey, entryCustomDataValue); - } else { - assert(false); } entryCustomDataKey = null; @@ -902,8 +886,6 @@ public class ImporterV4 extends Importer { ctxDeletedObject = null; return KdbContext.RootDeletedObjects; } else { - assert(false); - String contextName = ""; if (ctx != null) { contextName = ctx.name(); @@ -949,35 +931,24 @@ public class ImporterV4 extends Importer { } private void ReadUnknown(XmlPullParser xpp) throws XmlPullParserException, IOException { - assert(false); - if ( xpp.isEmptyElementTag() ) return; - - String unknownName = xpp.getName(); + ProcessNode(xpp); - while (xpp.next() != XmlPullParser.END_DOCUMENT ) { if ( xpp.getEventType() == XmlPullParser.END_TAG ) break; if ( xpp.getEventType() == XmlPullParser.START_TAG ) continue; ReadUnknown(xpp); } - - assert(xpp.getName().equals(unknownName)); - } private boolean ReadBool(XmlPullParser xpp, boolean bDefault) throws IOException, XmlPullParserException { - String str = ReadString(xpp); - - if ( str.equalsIgnoreCase("true") ) { - return true; - } else if ( str.equalsIgnoreCase("false") ) { - return false; - } else { - return bDefault; - } - } + String str = ReadString(xpp); + + return str.equalsIgnoreCase("true") + || !str.equalsIgnoreCase("false") + && bDefault; + } private UUID ReadUuid(XmlPullParser xpp) throws IOException, XmlPullParserException { String encoded = ReadString(xpp); @@ -1113,8 +1084,8 @@ public class ImporterV4 extends Importer { } private byte[] ProcessNode(XmlPullParser xpp) throws XmlPullParserException, IOException { - assert(xpp.getEventType() == XmlPullParser.START_TAG); - + //(xpp.getEventType() == XmlPullParser.START_TAG); + byte[] buf = null; if ( xpp.getAttributeCount() > 0 ) { diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 2b4a85657..810e5ba9e 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -402,7 +402,7 @@ public class PwDbV4Output extends PwDbOutput { } - private void writeObject(String key, ProtectedBinary value, boolean allowRef) throws IllegalArgumentException, IllegalStateException, IOException { + private void writeObject(String key, ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { assert(key != null && value != null); xml.startTag(null, PwDatabaseV4XML.ElemBinary); @@ -411,11 +411,8 @@ public class PwDbV4Output extends PwDbOutput { xml.endTag(null, PwDatabaseV4XML.ElemKey); xml.startTag(null, PwDatabaseV4XML.ElemValue); - String strRef = null; - if (allowRef) { - int ref = mPM.getBinPool().findKey(value); - strRef = Integer.toString(ref); - } + int ref = mPM.getBinPool().findKey(value); + String strRef = Integer.toString(ref); if (strRef != null) { xml.attribute(null, PwDatabaseV4XML.AttrRef, strRef); @@ -429,10 +426,10 @@ public class PwDbV4Output extends PwDbOutput { } private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { - int valLength = (int) value.length(); // TODO verify + + int valLength = (int) value.length(); // TODO verify if (valLength > 0) { byte[] buffer = new byte[valLength]; - value.getData().read(buffer, 0, valLength); // TODO Nullable if (value.isProtected()) { @@ -637,7 +634,7 @@ public class PwDbV4Output extends PwDbOutput { assert(binaries != null); for (Entry pair : binaries.entrySet()) { - writeObject(pair.getKey(), pair.getValue(), true); + writeObject(pair.getKey(), pair.getValue()); } } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 51c6bc961..2f1e373e2 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Brian Pellin, Jeremy Jamet / Kunzisoft. + * Copyright 2018 Jeremy Jamet / Kunzisoft. * * This file is part of KeePass DX. * @@ -45,10 +45,14 @@ public class ProtectedBinary implements Parcelable { } public long length() { - return size; + if (data != null) + return data.length; + if (dataFile != null) + return size; + return 0; } - public ProtectedBinary() { + private ProtectedBinary() { this.protect = false; this.data = null; this.dataFile = null; @@ -59,7 +63,10 @@ public class ProtectedBinary implements Parcelable { this.protect = enableProtection; this.data = data; this.dataFile = null; - this.size = data.length; + if (data != null) + this.size = data.length; + else + this.size = 0; } public ProtectedBinary(boolean enableProtection, File dataFile, int size) { @@ -69,7 +76,7 @@ public class ProtectedBinary implements Parcelable { this.size = size; } - public ProtectedBinary(Parcel in) { + private ProtectedBinary(Parcel in) { protect = in.readByte() != 0; in.readByteArray(data); dataFile = new File(in.readString()); diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java index 5d4cbfb88..1af163754 100644 --- a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java +++ b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java @@ -19,12 +19,9 @@ */ package com.kunzisoft.keepass.stream; -import com.kunzisoft.keepass.utils.Types; - import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.UUID; /** Little endian version of the DataInputStream @@ -33,7 +30,7 @@ import java.util.UUID; */ public class LEDataInputStream extends InputStream { - public static final long INT_TO_LONG_MASK = 0xffffffffL; + private static final long INT_TO_LONG_MASK = 0xffffffffL; private InputStream baseStream; @@ -129,7 +126,7 @@ public class LEDataInputStream extends InputStream { } public void readBytes(int length, ActionReadBytes actionReadBytes) throws IOException { - int bufferSize = 256; + int bufferSize = 256 * 3; // TODO Buffer size byte[] buffer = new byte[bufferSize]; int offset = 0; @@ -175,11 +172,11 @@ public class LEDataInputStream extends InputStream { * @return */ public static int readUShort( byte[] buf, int offset ) { - return (buf[offset + 0] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8); + return (buf[offset] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8); } public static long readLong( byte buf[], int offset ) { - return ((long)buf[offset + 0] & 0xFF) + (((long)buf[offset + 1] & 0xFF) << 8) + return ((long)buf[offset] & 0xFF) + (((long)buf[offset + 1] & 0xFF) << 8) + (((long)buf[offset + 2] & 0xFF) << 16) + (((long)buf[offset + 3] & 0xFF) << 24) + (((long)buf[offset + 4] & 0xFF) << 32) + (((long)buf[offset + 5] & 0xFF) << 40) + (((long)buf[offset + 6] & 0xFF) << 48) + (((long)buf[offset + 7] & 0xFF) << 56); @@ -209,14 +206,8 @@ public class LEDataInputStream extends InputStream { * @return */ public static int readInt( byte buf[], int offset ) { - return (buf[offset + 0] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8) + ((buf[offset + 2] & 0xFF) << 16) + return (buf[offset] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8) + ((buf[offset + 2] & 0xFF) << 16) + ((buf[offset + 3] & 0xFF) << 24); } - public UUID readUUID() throws IOException { - byte[] buf = readBytes(16); - - return Types.bytestoUUID(buf); - } - } diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index 70004d304..d783c93ac 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -25,18 +25,33 @@ import android.os.Parcelable; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; public class MemUtil { + + public static void copyStream(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[1024]; + int read; + try { + while ((read = in.read(buf)) != -1) { + out.write(buf, 0, read); + } + } catch (OutOfMemoryError error) { + throw new IOException(error); + } + } + public static byte[] decompress(byte[] input) throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(input); GZIPInputStream gzis = new GZIPInputStream(bais); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyStream(gzis, baos); + copyStream(gzis, baos); return baos.toByteArray(); } @@ -46,7 +61,7 @@ public class MemUtil { ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gzos = new GZIPOutputStream(baos); - Util.copyStream(bais, gzos); + copyStream(bais, gzos); gzos.close(); return baos.toByteArray(); diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/Util.java b/app/src/main/java/com/kunzisoft/keepass/utils/Util.java index c6b90e84d..ae8c899f0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/Util.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/Util.java @@ -27,16 +27,11 @@ import android.content.pm.ActivityInfo; import android.content.res.Configuration; import android.graphics.Typeface; import android.net.Uri; -import android.util.TypedValue; import android.widget.EditText; import android.widget.TextView; import com.kunzisoft.keepass.R; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - public class Util { public static void gotoUrl(Context context, String url) throws ActivityNotFoundException { @@ -49,20 +44,8 @@ public class Util { public static void gotoUrl(Context context, int resId) throws ActivityNotFoundException { gotoUrl(context, context.getString(resId)); } - - public static void copyStream(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[1024]; - int read; - try { - while ((read = in.read(buf)) != -1) { - out.write(buf, 0, read); - } - } catch (OutOfMemoryError error) { - throw new IOException(error); - } - } - /** + /** * Replace font by monospace, must be called after seText() */ public static void applyFontVisibilityTo(final Context context, final TextView textView) { From dc23c6a445dd8796ff2b6f0d91a9c19cd4c91d6c Mon Sep 17 00:00:00 2001 From: J-Jamet Date: Tue, 18 Sep 2018 10:11:44 +0200 Subject: [PATCH 8/8] Optimize memory pool and revert algo --- .../keepass/database/load/ImporterV4.java | 42 +++++--- .../save/PwDbInnerHeaderOutputV4.java | 20 +--- .../keepass/database/save/PwDbV4Output.java | 102 +++++++++++------- .../com/kunzisoft/keepass/utils/MemUtil.java | 83 +++++++++++++- 4 files changed, 176 insertions(+), 71 deletions(-) diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 221aece5d..83d49428c 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -23,7 +23,6 @@ import com.kunzisoft.keepass.R; import com.kunzisoft.keepass.crypto.CipherFactory; import com.kunzisoft.keepass.crypto.PwStreamCipherFactory; import com.kunzisoft.keepass.crypto.engine.CipherEngine; -import com.kunzisoft.keepass.database.BinaryPool; import com.kunzisoft.keepass.database.ITimeLogger; import com.kunzisoft.keepass.database.PwCompressionAlgorithm; import com.kunzisoft.keepass.database.PwDatabase; @@ -211,7 +210,10 @@ public class ImporterV4 extends Importer { while(true) { if (!ReadInnerHeader(lis, header)) break; } + } + private String getUnusedCacheFileName() { + return String.valueOf(db.getBinPool().findUnusedKey()); } private boolean ReadInnerHeader(LEDataInputStream lis, PwDbHeaderV4 header) throws IOException { @@ -241,15 +243,14 @@ public class ImporterV4 extends Importer { byte flag = lis.readBytes(1)[0]; boolean protectedFlag = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != PwDbHeaderV4.KdbxBinaryFlags.None; + int byteLength = size - 1; // Read in a file - BinaryPool binaryPool = db.getBinPool(); - int binaryKey = binaryPool.findUnusedKey(); - File file = new File(streamDir, String.valueOf(binaryKey)); + File file = new File(streamDir, getUnusedCacheFileName()); try (FileOutputStream outputStream = new FileOutputStream(file)) { - lis.readBytes(size - 1, outputStream::write); + lis.readBytes(byteLength, outputStream::write); } - ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, size -1); - binaryPool.add(protectedBinary); + ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, byteLength); + db.getBinPool().add(protectedBinary); break; default: @@ -288,9 +289,9 @@ public class ImporterV4 extends Importer { } private static final long DEFAULT_HISTORY_DAYS = 365; - + private boolean readNextNode = true; - private Stack ctxGroups = new Stack(); + private Stack ctxGroups = new Stack<>(); private PwGroupV4 ctxGroup = null; private PwEntryV4 ctxEntry = null; private String ctxStringName = null; @@ -336,7 +337,7 @@ public class ImporterV4 extends Importer { ctxGroups.clear(); KdbContext ctx = KdbContext.Null; - + readNextNode = true; while (true) { @@ -1027,7 +1028,19 @@ public class ImporterV4 extends Importer { return new ProtectedString(false, ReadString(xpp)); } - + + private ProtectedBinary createProtectedBinaryFromData(boolean protection, byte[] data) throws IOException { + if (data.length > MemUtil.BUFFER_SIZE_BYTES) { + File file = new File(streamDir, getUnusedCacheFileName()); + try (FileOutputStream outputStream = new FileOutputStream(file)) { + outputStream.write(data); + } + return new ProtectedBinary(protection, file, data.length); + } else { + return new ProtectedBinary(protection, data); + } + } + private ProtectedBinary ReadProtectedBinary(XmlPullParser xpp) throws XmlPullParserException, IOException { String ref = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrRef); if (ref != null) { @@ -1045,8 +1058,9 @@ public class ImporterV4 extends Importer { byte[] buf = ProcessNode(xpp); - if ( buf != null ) - return new ProtectedBinary(true, buf); + if ( buf != null ) { + createProtectedBinaryFromData(true, buf); + } String base64 = ReadString(xpp); if ( base64.length() == 0 ) @@ -1058,7 +1072,7 @@ public class ImporterV4 extends Importer { data = MemUtil.decompress(data); } - return new ProtectedBinary(false, data); + return createProtectedBinaryFromData(false, data); } private String ReadString(XmlPullParser xpp) throws IOException, XmlPullParserException { diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java index b54eb6d92..a362ab7c0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java @@ -23,9 +23,9 @@ import com.kunzisoft.keepass.database.PwDatabaseV4; import com.kunzisoft.keepass.database.PwDbHeaderV4; import com.kunzisoft.keepass.database.security.ProtectedBinary; import com.kunzisoft.keepass.stream.LEDataOutputStream; +import com.kunzisoft.keepass.utils.MemUtil; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; public class PwDbInnerHeaderOutputV4 { @@ -60,26 +60,8 @@ public class PwDbInnerHeaderOutputV4 { los.writeInt((int) protectedBinary.length() + 1); // TODO verify los.write(flag); - /* MemUtil.readBytes(protectedBinary.getData(), - (int) protectedBinary.length(), buffer -> los.write(buffer)); - */ - - byte[] buffer = new byte[3 * 256]; // TODO buffer generalize - InputStream fileInputStream = protectedBinary.getData(); // TODO Nullable - // To create the last buffer who is smaller - long numberOfFullBuffer = protectedBinary.length() / buffer.length; - long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; - int read = 0; - //if (protectedBinary.length() > 0) { - while (read < protectedBinary.length()) { - // Create the last smaller buffer - if (read >= sizeOfFullBuffers) - buffer = new byte[(int) (protectedBinary.length() % buffer.length)]; - read += fileInputStream.read(buffer, 0, buffer.length); - los.write(buffer); - } } los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.EndOfHeader); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 810e5ba9e..d697ae517 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -394,9 +394,8 @@ public class PwDbV4Output extends PwDbOutput { if (!isHistory) { writeList(PwDatabaseV4XML.ElemHistory, entry.getHistory(), true); - } else { - assert(entry.sizeOfHistory() == 0); } + // else entry.sizeOfHistory() == 0 xml.endTag(null, PwDatabaseV4XML.ElemEntry); } @@ -424,54 +423,85 @@ public class PwDbV4Output extends PwDbOutput { xml.endTag(null, PwDatabaseV4XML.ElemBinary); } - - private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { - int valLength = (int) value.length(); // TODO verify - if (valLength > 0) { - byte[] buffer = new byte[valLength]; - value.getData().read(buffer, 0, valLength); // TODO Nullable + /* + TODO Make with pipe + private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + try (InputStream inputStream = value.getData()) { + if (inputStream == null) { + Log.e(TAG, "Can't write a null input stream."); + return; + } if (value.isProtected()) { xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); - byte[] encoded = new byte[valLength]; - randomStream.processBytes(buffer, 0, valLength, encoded, 0); - xml.text(String.valueOf(Base64Coder.encode(encoded))); + try (InputStream cypherInputStream = + IOUtil.pipe(inputStream, + o -> new org.spongycastle.crypto.io.CipherOutputStream(o, randomStream))) { + writeInputStreamInBase64(cypherInputStream); + } } else { if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); - byte[] compressData = MemUtil.compress(buffer); - //InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, valLength); // TODO buffer generalize - //gzipInputStream.read(buffer, 0, valLength); - xml.text(String.valueOf(Base64Coder.encode(compressData))); // TODO Compress all - /* - // TODO be a problem not the same length - MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() { - @Override - public void doAction(byte[] buffer) throws IOException { - xml.text(String.valueOf(Base64Coder.encode(buffer))); - } - }); - */ - } - else { - xml.text(String.valueOf(Base64Coder.encode(buffer))); - /* - // TODO Debug - MemUtil.readBytes(value.getData(), - (int) value.length(), - buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer)))); - */ + try (InputStream gZipInputStream = + IOUtil.pipe(inputStream, GZIPOutputStream::new, (int) value.length())) { + writeInputStreamInBase64(gZipInputStream); + } + + } else { + writeInputStreamInBase64(inputStream); } } - - // TODO remove buffer } } - + + private void writeInputStreamInBase64(InputStream inputStream) throws IOException { + try (InputStream base64InputStream = + IOUtil.pipe(inputStream, + o -> new Base64OutputStream(o, DEFAULT))) { + MemUtil.readBytes(base64InputStream, + buffer -> xml.text(Arrays.toString(buffer))); + } + } + //*/ + + //* + private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + + int valLength = (int) value.length(); + if (valLength > 0) { + byte[] buffer = new byte[valLength]; + if (valLength == value.getData().read(buffer, 0, valLength)) { + + if (value.isProtected()) { + xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); + + byte[] encoded = new byte[valLength]; + randomStream.processBytes(buffer, 0, valLength, encoded, 0); + xml.text(String.valueOf(Base64Coder.encode(encoded))); + + } else { + if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); + + byte[] compressData = MemUtil.compress(buffer); + xml.text(String.valueOf(Base64Coder.encode(compressData))); + + } else { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + } + } + } else { + Log.e(TAG, "Unable to read the stream of the protected binary"); + } + } + } + //*/ + private void writeObject(String name, String value, boolean filterXmlChars) throws IllegalArgumentException, IllegalStateException, IOException { assert(name != null && value != null); diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index d783c93ac..42012767c 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -21,12 +21,21 @@ package com.kunzisoft.keepass.utils; import android.os.Parcel; import android.os.Parcelable; +import android.support.annotation.NonNull; +import android.util.Log; + +import com.kunzisoft.keepass.stream.ActionReadBytes; + +import org.apache.commons.io.IOUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -34,8 +43,11 @@ import java.util.zip.GZIPOutputStream; public class MemUtil { + private static final String TAG = MemUtil.class.getName(); + public static final int BUFFER_SIZE_BYTES = 3 * 128; + public static void copyStream(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[1024]; + byte[] buf = new byte[BUFFER_SIZE_BYTES]; int read; try { while ((read = in.read(buf)) != -1) { @@ -46,6 +58,40 @@ public class MemUtil { } } + public static void readBytes(@NonNull InputStream inputStream, ActionReadBytes actionReadBytes) + throws IOException { + byte[] buffer = new byte[MemUtil.BUFFER_SIZE_BYTES]; + int read = 0; + while (read != -1) { + read = inputStream.read(buffer, 0, buffer.length); + if (read != -1) { + byte[] optimizedBuffer; + if (buffer.length == read) { + optimizedBuffer = buffer; + } else { + optimizedBuffer = Arrays.copyOf(buffer, read); + } + actionReadBytes.doAction(optimizedBuffer); + } + } + + /* + byte[] buffer = new byte[BUFFER_SIZE_BYTES]; + // To create the last buffer who is smaller + long numberOfFullBuffer = length / buffer.length; + long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; + int read = 0; + //if (protectedBinary.length() > 0) { + while (read < length) { + // Create the last smaller buffer + if (read >= sizeOfFullBuffers) + buffer = new byte[(int) (length % buffer.length)]; + read += inputStream.read(buffer, 0, buffer.length); + actionReadBytes.doAction(buffer); + } + //*/ + } + public static byte[] decompress(byte[] input) throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(input); GZIPInputStream gzis = new GZIPInputStream(bais); @@ -67,7 +113,40 @@ public class MemUtil { return baos.toByteArray(); } - // For writing to a Parcel + /** + * Compresses the input data using GZip and outputs the compressed data. + * + * @param input + * An {@link InputStream} containing the input raw data. + * + * @return An {@link InputStream} to the compressed data. + */ + public static InputStream compress(final InputStream input) { + final PipedInputStream compressedDataStream = new PipedInputStream(3 * 128); + Log.d(TAG, "About to compress input data using gzip asynchronously..."); + PipedOutputStream compressionOutput; + GZIPOutputStream gzipCompressedDataStream = null; + try { + compressionOutput = new PipedOutputStream(compressedDataStream); + gzipCompressedDataStream = new GZIPOutputStream(compressionOutput); + IOUtils.copy(input, gzipCompressedDataStream); + Log.e(TAG, "Successfully compressed input data using gzip."); + } catch (IOException e) { + Log.e(TAG, "Failed to compress input data.", e); + } finally { + if (gzipCompressedDataStream != null) { + try { + gzipCompressedDataStream.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close gzip output stream.", e); + } + } + } + return compressedDataStream; + } + + + // For writing to a Parcel public static void writeParcelableMap( Parcel parcel, int flags, Map map) { parcel.writeInt(map.size());