diff --git a/app/build.gradle b/app/build.gradle index 350aad3f8..ad2a12dfd 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -103,8 +103,11 @@ dependencies { annotationProcessor "com.github.hotchemi:permissionsdispatcher-processor:$permissionDispatcherVersion" // Apache Commons Collections implementation 'commons-collections:commons-collections:3.2.1' + implementation 'org.apache.commons:commons-io:1.3.2' // Base64 implementation 'biz.source_code:base64coder:2010-12-19' + // IO-Extras + implementation 'com.github.davidmoten:io-extras:0.1' implementation 'com.google.code.gson:gson:2.8.4' implementation 'com.google.guava:guava:23.0-android' // Icon pack diff --git a/app/src/main/java/com/kunzisoft/keepass/app/App.java b/app/src/main/java/com/kunzisoft/keepass/app/App.java index 6a861dcde..346fdd1e4 100644 --- a/app/src/main/java/com/kunzisoft/keepass/app/App.java +++ b/app/src/main/java/com/kunzisoft/keepass/app/App.java @@ -92,7 +92,7 @@ public class App extends MultiDexApplication { @Override public void onTerminate() { if ( db != null ) { - db.clear(); + db.clear(getApplicationContext()); } super.onTerminate(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java index 84fed9719..c256d5fd9 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/BinaryPool.java @@ -51,6 +51,8 @@ public class BinaryPool { } public void clear() { + for (Entry entry: pool.entrySet()) + entry.getValue().clear(); pool.clear(); } @@ -63,30 +65,36 @@ public class BinaryPool { @Override public boolean operate(PwEntryV4 entry) { for (PwEntryV4 histEntry : entry.getHistory()) { - poolAdd(histEntry.getBinaries()); + add(histEntry.getBinaries()); } - poolAdd(entry.getBinaries()); + add(entry.getBinaries()); return true; } } - private void poolAdd(Map dict) { + private void add(Map dict) { for (ProtectedBinary pb : dict.values()) { - poolAdd(pb); + add(pb); } } - public void poolAdd(ProtectedBinary pb) { - assert(pb != null); + public void add(ProtectedBinary pb) { + assert(pb != null); + if (findKey(pb) != -1) return; - if (poolFind(pb) != -1) return; - - pool.put(pool.size(), pb); + pool.put(findUnusedKey(), pb); } + + public int findUnusedKey() { + int unusedKey = pool.size(); + while(get(unusedKey) != null) + unusedKey++; + return unusedKey; + } - public int poolFind(ProtectedBinary pb) { + public int findKey(ProtectedBinary pb) { for (Entry pair : pool.entrySet()) { if (pair.getValue().equals(pb)) return pair.getKey(); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/Database.java b/app/src/main/java/com/kunzisoft/keepass/database/Database.java index 6dd484e49..ce5c4ce50 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/Database.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/Database.java @@ -39,6 +39,8 @@ import com.kunzisoft.keepass.icons.IconDrawableFactory; import com.kunzisoft.keepass.tasks.ProgressTaskUpdater; import com.kunzisoft.keepass.utils.UriUtil; +import org.apache.commons.io.FileUtils; + import java.io.BufferedInputStream; import java.io.File; import java.io.FileNotFoundException; @@ -58,7 +60,7 @@ public class Database { private static final String TAG = Database.class.getName(); - private PwDatabase pm; + private PwDatabase pwDatabase; private Uri mUri; private SearchDbHelper searchHelper; private boolean readOnly = false; @@ -69,11 +71,11 @@ public class Database { private boolean loaded = false; public PwDatabase getPwDatabase() { - return pm; + return pwDatabase; } public void setPwDatabase(PwDatabase pm) { - this.pm = pm; + this.pwDatabase = pm; } public void setUri(Uri mUri) { @@ -151,24 +153,25 @@ public class Database { // We'll end up reading 8 bytes to identify the header. Might as well use two extra. bis.mark(10); - Importer databaseImporter = ImporterFactory.createImporter(bis, debug); + // Get the file directory to save the attachments + Importer databaseImporter = ImporterFactory.createImporter(bis, ctx.getFilesDir(), debug); bis.reset(); // Return to the start - pm = databaseImporter.openDatabase(bis, password, keyFileInputStream, progressTaskUpdater); - if ( pm != null ) { + pwDatabase = databaseImporter.openDatabase(bis, password, keyFileInputStream, progressTaskUpdater); + if ( pwDatabase != null ) { try { - switch (pm.getVersion()) { + switch (pwDatabase.getVersion()) { case V3: - PwGroupV3 rootV3 = ((PwDatabaseV3) pm).getRootGroup(); - ((PwDatabaseV3) pm).populateGlobals(rootV3); - passwordEncodingError = !pm.validatePasswordEncoding(password); + PwGroupV3 rootV3 = ((PwDatabaseV3) pwDatabase).getRootGroup(); + ((PwDatabaseV3) pwDatabase).populateGlobals(rootV3); + passwordEncodingError = !pwDatabase.validatePasswordEncoding(password); searchHelper = new SearchDbHelper.SearchDbHelperV3(ctx); break; case V4: - PwGroupV4 rootV4 = ((PwDatabaseV4) pm).getRootGroup(); - ((PwDatabaseV4) pm).populateGlobals(rootV4); - passwordEncodingError = !pm.validatePasswordEncoding(password); + PwGroupV4 rootV4 = ((PwDatabaseV4) pwDatabase).getRootGroup(); + ((PwDatabaseV4) pwDatabase).populateGlobals(rootV4); + passwordEncodingError = !pwDatabase.validatePasswordEncoding(password); searchHelper = new SearchDbHelper.SearchDbHelperV4(ctx); break; } @@ -187,11 +190,11 @@ public class Database { public PwGroup search(String str, int max) { if (searchHelper == null) { return null; } try { - switch (pm.getVersion()) { + switch (pwDatabase.getVersion()) { case V3: - return ((SearchDbHelper.SearchDbHelperV3) searchHelper).search(((PwDatabaseV3) pm), str, max); + return ((SearchDbHelper.SearchDbHelperV3) searchHelper).search(((PwDatabaseV3) pwDatabase), str, max); case V4: - return ((SearchDbHelper.SearchDbHelperV4) searchHelper).search(((PwDatabaseV4) pm), str, max); + return ((SearchDbHelper.SearchDbHelperV4) searchHelper).search(((PwDatabaseV4) pwDatabase), str, max); } } catch (Exception e) { Log.e(TAG, "Search can't be performed with this SearchHelper", e); @@ -261,7 +264,7 @@ public class Database { FileOutputStream fos = null; try { fos = new FileOutputStream(tempFile); - PwDbOutput pmo = PwDbOutput.getInstance(pm, fos); + PwDbOutput pmo = PwDbOutput.getInstance(pwDatabase, fos); if (pmo != null) pmo.output(); } catch (Exception e) { @@ -289,7 +292,7 @@ public class Database { OutputStream os = null; try { os = ctx.getContentResolver().openOutputStream(uri); - PwDbOutput pmo = PwDbOutput.getInstance(pm, os); + PwDbOutput pmo = PwDbOutput.getInstance(pwDatabase, os); if (pmo != null) pmo.output(); } catch (Exception e) { @@ -304,10 +307,19 @@ public class Database { } // TODO Clear database when lock broadcast is receive in backstage - public void clear() { + public void clear(Context context) { drawFactory.clearCache(); + // Delete the cache of the database if present + if (pwDatabase != null) + pwDatabase.clearCache(); + // In all cases, delete all the files in the temp dir + try { + FileUtils.cleanDirectory(context.getFilesDir()); + } catch (IOException e) { + Log.e(TAG, "Unable to clear the directory cache.", e); + } - pm = null; + pwDatabase = null; mUri = null; loaded = false; passwordEncodingError = false; @@ -545,7 +557,7 @@ public class Database { case V4: newPwGroup = new PwGroupV4((PwGroupV4) parent); } - newPwGroup.setId(pm.newGroupId()); + newPwGroup.setId(pwDatabase.newGroupId()); } catch (Exception e) { Log.e(TAG, "This version of PwGroup can't be created", e); } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java index cd2a797ae..09b324178 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabase.java @@ -21,7 +21,7 @@ package com.kunzisoft.keepass.database; import com.kunzisoft.keepass.database.exception.InvalidKeyFileException; import com.kunzisoft.keepass.database.exception.KeyFileEmptyException; -import com.kunzisoft.keepass.utils.Util; +import com.kunzisoft.keepass.utils.MemUtil; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -128,7 +128,7 @@ public abstract class PwDatabase assert(keyInputStream != null); ByteArrayOutputStream bos = new ByteArrayOutputStream(); - Util.copyStream(keyInputStream, bos); + MemUtil.copyStream(keyInputStream, bos); byte[] keyData = bos.toByteArray(); ByteArrayInputStream bis = new ByteArrayInputStream(keyData); @@ -430,4 +430,6 @@ public abstract class PwDatabase */ public abstract void initNew(String dbPath); + public abstract void clearCache(); + } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java index 3c813614e..8be18ff21 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV3.java @@ -376,4 +376,7 @@ public class PwDatabaseV3 extends PwDatabase { return !(omitBackup && isBackup(group)); } + + @Override + public void clearCache() {} } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java index 512a8e644..bf7490bf6 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwDatabaseV4.java @@ -754,4 +754,8 @@ public class PwDatabaseV4 extends PwDatabase { return filename.substring(0, lastExtDot); } + @Override + public void clearCache() { + binPool.clear(); + } } \ No newline at end of file diff --git a/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java b/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java index 921a44141..677db7f26 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/PwEntryV4.java @@ -93,8 +93,7 @@ public class PwEntryV4 extends PwEntry implements ITimeLogger { parentGroupLastMod = in.readParcelable(PwDate.class.getClassLoader()); customData = MemUtil.readStringParcelableMap(in); fields = in.readParcelable(ExtraFields.class.getClassLoader()); - // TODO binaries takes too much memory for parcelable - // binaries = MemUtil.readStringParcelableMap(in, ProtectedBinary.class); + binaries = MemUtil.readStringParcelableMap(in, ProtectedBinary.class); foregroundColor = in.readString(); backgroupColor = in.readString(); overrideURL = in.readString(); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java index 385159fd7..02cefcb7e 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterFactory.java @@ -24,15 +24,16 @@ import com.kunzisoft.keepass.database.PwDbHeaderV4; import com.kunzisoft.keepass.database.exception.InvalidDBSignatureException; import com.kunzisoft.keepass.stream.LEDataInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; public class ImporterFactory { - public static Importer createImporter(InputStream is) throws InvalidDBSignatureException, IOException { - return createImporter(is, false); + public static Importer createImporter(InputStream is, File streamDir) throws InvalidDBSignatureException, IOException { + return createImporter(is, streamDir,false); } - public static Importer createImporter(InputStream is, boolean debug) throws InvalidDBSignatureException, IOException { + public static Importer createImporter(InputStream is, File streamDir, boolean debug) throws InvalidDBSignatureException, IOException { int sig1 = LEDataInputStream.readInt(is); int sig2 = LEDataInputStream.readInt(is); @@ -43,7 +44,7 @@ public class ImporterFactory { return new ImporterV3(); } else if ( PwDbHeaderV4.matchesHeader(sig1, sig2) ) { - return new ImporterV4(); + return new ImporterV4(streamDir); } throw new InvalidDBSignatureException(); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java index 8f395414d..42e4c1cdd 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/load/ImporterV4.java @@ -54,6 +54,8 @@ import org.xmlpull.v1.XmlPullParser; import org.xmlpull.v1.XmlPullParserException; import org.xmlpull.v1.XmlPullParserFactory; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -62,10 +64,8 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.text.ParseException; import java.util.Arrays; -import java.util.Calendar; import java.util.Date; import java.util.Stack; -import java.util.TimeZone; import java.util.UUID; import java.util.zip.GZIPInputStream; @@ -80,12 +80,11 @@ public class ImporterV4 extends Importer { private PwDatabaseV4 db; private byte[] hashOfHeader = null; - private byte[] pbHeader = null; private long version; - Calendar utcCal; + private File streamDir; - public ImporterV4() { - utcCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + public ImporterV4(File streamDir) { + this.streamDir = streamDir; } @Override @@ -111,7 +110,7 @@ public class ImporterV4 extends Importer { version = header.getVersion(); hashOfHeader = hh.hash; - pbHeader = hh.header; + byte[] pbHeader = hh.header; db.retrieveMasterKey(password, keyInputStream); db.makeFinalKey(header.masterSeed); @@ -185,7 +184,6 @@ public class ImporterV4 extends Importer { } if ( header.innerRandomStreamKey == null ) { - assert(false); throw new IOException("Invalid stream key."); } @@ -212,7 +210,10 @@ public class ImporterV4 extends Importer { while(true) { if (!ReadInnerHeader(lis, header)) break; } + } + private String getUnusedCacheFileName() { + return String.valueOf(db.getBinPool().findUnusedKey()); } private boolean ReadInnerHeader(LEDataInputStream lis, PwDbHeaderV4 header) throws IOException { @@ -223,7 +224,8 @@ public class ImporterV4 extends Importer { byte[] data = new byte[0]; if (size > 0) { - data = lis.readBytes(size); + if (fieldId != PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary) + data = lis.readBytes(size); } boolean result = true; @@ -238,22 +240,20 @@ public class ImporterV4 extends Importer { header.innerRandomStreamKey = data; break; case PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary: - if (data.length < 1) throw new IOException("Invalid binary format"); - byte flag = data[0]; - boolean prot = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != - PwDbHeaderV4.KdbxBinaryFlags.None; - - byte[] bin = new byte[data.length - 1]; - System.arraycopy(data, 1, bin, 0, data.length-1); - ProtectedBinary pb = new ProtectedBinary(prot, bin); - db.getBinPool().poolAdd(pb); - - if (prot) { - Arrays.fill(data, (byte)0); + byte flag = lis.readBytes(1)[0]; + boolean protectedFlag = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) != + PwDbHeaderV4.KdbxBinaryFlags.None; + int byteLength = size - 1; + // Read in a file + File file = new File(streamDir, getUnusedCacheFileName()); + try (FileOutputStream outputStream = new FileOutputStream(file)) { + lis.readBytes(byteLength, outputStream::write); } + ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, byteLength); + db.getBinPool().add(protectedBinary); break; + default: - assert(false); break; } @@ -289,9 +289,9 @@ public class ImporterV4 extends Importer { } private static final long DEFAULT_HISTORY_DAYS = 365; - + private boolean readNextNode = true; - private Stack ctxGroups = new Stack(); + private Stack ctxGroups = new Stack<>(); private PwGroupV4 ctxGroup = null; private PwEntryV4 ctxEntry = null; private String ctxStringName = null; @@ -337,7 +337,7 @@ public class ImporterV4 extends Importer { ctxGroups.clear(); KdbContext ctx = KdbContext.Null; - + readNextNode = true; while (true) { @@ -348,20 +348,17 @@ public class ImporterV4 extends Importer { } switch ( xpp.getEventType() ) { - case XmlPullParser.START_TAG: - ctx = ReadXmlElement(ctx, xpp); - break; - - case XmlPullParser.END_TAG: - ctx = EndXmlElement(ctx, xpp); - break; + case XmlPullParser.START_TAG: + ctx = ReadXmlElement(ctx, xpp); + break; - default: - assert(false); - break; - + case XmlPullParser.END_TAG: + ctx = EndXmlElement(ctx, xpp); + break; + + default: + break; } - } // Error checks @@ -533,8 +530,8 @@ public class ImporterV4 extends Importer { case Root: if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemGroup) ) { - assert(ctxGroups.size() == 0); - if ( ctxGroups.size() != 0 ) throw new IOException("Group list should be empty."); + if ( ctxGroups.size() != 0 ) + throw new IOException("Group list should be empty."); db.setRootGroup(new PwGroupV4()); ctxGroups.push(db.getRootGroup()); @@ -633,8 +630,6 @@ public class ImporterV4 extends Importer { } else if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemCustomData)) { return SwitchContext(ctx, KdbContext.EntryCustomData, xpp); } else if ( name.equalsIgnoreCase(PwDatabaseV4XML.ElemHistory) ) { - assert(!entryInHistory); - if ( ! entryInHistory ) { ctxHistoryBase = ctxEntry; return SwitchContext(ctx, KdbContext.EntryHistory, xpp); @@ -774,8 +769,8 @@ public class ImporterV4 extends Importer { } private KdbContext EndXmlElement(KdbContext ctx, XmlPullParser xpp) throws XmlPullParserException { - assert(xpp.getEventType() == XmlPullParser.END_TAG); - + // (xpp.getEventType() == XmlPullParser.END_TAG); + String name = xpp.getName(); if ( ctx == KdbContext.KeePassFile && name.equalsIgnoreCase(PwDatabaseV4XML.ElemDocNode) ) { return KdbContext.Null; @@ -792,7 +787,7 @@ public class ImporterV4 extends Importer { PwIconCustom icon = new PwIconCustom(customIconID, customIconData); db.addCustomIcon(icon); db.getIconFactory().put(icon); - } else assert(false); + } customIconID = PwDatabase.UUID_ZERO; customIconData = null; @@ -805,7 +800,7 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.CustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem) ) { if ( customDataKey != null && customDataValue != null) { db.putCustomData(customDataKey, customDataValue); - } else assert(false); + } customDataKey = null; customDataValue = null; @@ -832,8 +827,6 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.GroupCustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem)) { if (groupCustomDataKey != null && groupCustomDataValue != null) { ctxGroup.putCustomData(groupCustomDataKey, groupCustomDataKey); - } else { - assert(false); } groupCustomDataKey = null; @@ -879,8 +872,6 @@ public class ImporterV4 extends Importer { } else if ( ctx == KdbContext.EntryCustomDataItem && name.equalsIgnoreCase(PwDatabaseV4XML.ElemStringDictExItem)) { if (entryCustomDataKey != null && entryCustomDataValue != null) { ctxEntry.putCustomData(entryCustomDataKey, entryCustomDataValue); - } else { - assert(false); } entryCustomDataKey = null; @@ -896,8 +887,6 @@ public class ImporterV4 extends Importer { ctxDeletedObject = null; return KdbContext.RootDeletedObjects; } else { - assert(false); - String contextName = ""; if (ctx != null) { contextName = ctx.name(); @@ -943,35 +932,24 @@ public class ImporterV4 extends Importer { } private void ReadUnknown(XmlPullParser xpp) throws XmlPullParserException, IOException { - assert(false); - if ( xpp.isEmptyElementTag() ) return; - - String unknownName = xpp.getName(); + ProcessNode(xpp); - while (xpp.next() != XmlPullParser.END_DOCUMENT ) { if ( xpp.getEventType() == XmlPullParser.END_TAG ) break; if ( xpp.getEventType() == XmlPullParser.START_TAG ) continue; ReadUnknown(xpp); } - - assert(xpp.getName().equals(unknownName)); - } private boolean ReadBool(XmlPullParser xpp, boolean bDefault) throws IOException, XmlPullParserException { - String str = ReadString(xpp); - - if ( str.equalsIgnoreCase("true") ) { - return true; - } else if ( str.equalsIgnoreCase("false") ) { - return false; - } else { - return bDefault; - } - } + String str = ReadString(xpp); + + return str.equalsIgnoreCase("true") + || !str.equalsIgnoreCase("false") + && bDefault; + } private UUID ReadUuid(XmlPullParser xpp) throws IOException, XmlPullParserException { String encoded = ReadString(xpp); @@ -1050,7 +1028,19 @@ public class ImporterV4 extends Importer { return new ProtectedString(false, ReadString(xpp)); } - + + private ProtectedBinary createProtectedBinaryFromData(boolean protection, byte[] data) throws IOException { + if (data.length > MemUtil.BUFFER_SIZE_BYTES) { + File file = new File(streamDir, getUnusedCacheFileName()); + try (FileOutputStream outputStream = new FileOutputStream(file)) { + outputStream.write(data); + } + return new ProtectedBinary(protection, file, data.length); + } else { + return new ProtectedBinary(protection, data); + } + } + private ProtectedBinary ReadProtectedBinary(XmlPullParser xpp) throws XmlPullParserException, IOException { String ref = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrRef); if (ref != null) { @@ -1068,10 +1058,13 @@ public class ImporterV4 extends Importer { byte[] buf = ProcessNode(xpp); - if ( buf != null ) return new ProtectedBinary(true, buf); + if ( buf != null ) { + createProtectedBinaryFromData(true, buf); + } String base64 = ReadString(xpp); - if ( base64.length() == 0 ) return ProtectedBinary.EMPTY; + if ( base64.length() == 0 ) + return ProtectedBinary.EMPTY; byte[] data = Base64Coder.decode(base64); @@ -1079,7 +1072,7 @@ public class ImporterV4 extends Importer { data = MemUtil.decompress(data); } - return new ProtectedBinary(false, data); + return createProtectedBinaryFromData(false, data); } private String ReadString(XmlPullParser xpp) throws IOException, XmlPullParserException { @@ -1105,13 +1098,14 @@ public class ImporterV4 extends Importer { } private byte[] ProcessNode(XmlPullParser xpp) throws XmlPullParserException, IOException { - assert(xpp.getEventType() == XmlPullParser.START_TAG); - + //(xpp.getEventType() == XmlPullParser.START_TAG); + byte[] buf = null; if ( xpp.getAttributeCount() > 0 ) { String protect = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrProtected); if ( protect != null && protect.equalsIgnoreCase(PwDatabaseV4XML.ValTrue) ) { + // TODO stream for encrypted data String encrypted = ReadStringRaw(xpp); if ( encrypted.length() > 0 ) { diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java index 89dc3145e..a362ab7c0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbInnerHeaderOutputV4.java @@ -23,10 +23,10 @@ import com.kunzisoft.keepass.database.PwDatabaseV4; import com.kunzisoft.keepass.database.PwDbHeaderV4; import com.kunzisoft.keepass.database.security.ProtectedBinary; import com.kunzisoft.keepass.stream.LEDataOutputStream; +import com.kunzisoft.keepass.utils.MemUtil; import java.io.IOException; import java.io.OutputStream; -import java.util.Arrays; public class PwDbInnerHeaderOutputV4 { private PwDatabaseV4 db; @@ -50,19 +50,18 @@ public class PwDbInnerHeaderOutputV4 { los.writeInt(streamKeySize); los.write(header.innerRandomStreamKey); - for (ProtectedBinary bin : db.getBinPool().binaries()) { + for (ProtectedBinary protectedBinary : db.getBinPool().binaries()) { byte flag = PwDbHeaderV4.KdbxBinaryFlags.None; - if (bin.isProtected()) { + if (protectedBinary.isProtected()) { flag |= PwDbHeaderV4.KdbxBinaryFlags.Protected; } - byte[] binData = bin.getData(); los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.Binary); - los.writeInt(bin.length() + 1); + los.writeInt((int) protectedBinary.length() + 1); // TODO verify los.write(flag); - los.write(binData); - Arrays.fill(binData, (byte)0); + MemUtil.readBytes(protectedBinary.getData(), + buffer -> los.write(buffer)); } los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.EndOfHeader); diff --git a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java index 53b4f5701..d697ae517 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/save/PwDbV4Output.java @@ -394,15 +394,14 @@ public class PwDbV4Output extends PwDbOutput { if (!isHistory) { writeList(PwDatabaseV4XML.ElemHistory, entry.getHistory(), true); - } else { - assert(entry.sizeOfHistory() == 0); } + // else entry.sizeOfHistory() == 0 xml.endTag(null, PwDatabaseV4XML.ElemEntry); } - private void writeObject(String key, ProtectedBinary value, boolean allowRef) throws IllegalArgumentException, IllegalStateException, IOException { + private void writeObject(String key, ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { assert(key != null && value != null); xml.startTag(null, PwDatabaseV4XML.ElemBinary); @@ -411,11 +410,8 @@ public class PwDbV4Output extends PwDbOutput { xml.endTag(null, PwDatabaseV4XML.ElemKey); xml.startTag(null, PwDatabaseV4XML.ElemValue); - String strRef = null; - if (allowRef) { - int ref = mPM.getBinPool().poolFind(value); - strRef = Integer.toString(ref); - } + int ref = mPM.getBinPool().findKey(value); + String strRef = Integer.toString(ref); if (strRef != null) { xml.attribute(null, PwDatabaseV4XML.AttrRef, strRef); @@ -427,33 +423,85 @@ public class PwDbV4Output extends PwDbOutput { xml.endTag(null, PwDatabaseV4XML.ElemBinary); } - + + /* + TODO Make with pipe private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { - if (value.isProtected()) { - xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); - - int valLength = value.length(); - if (valLength > 0) { - byte[] encoded = new byte[valLength]; - randomStream.processBytes(value.getData(), 0, valLength, encoded, 0); - - xml.text(String.valueOf(Base64Coder.encode(encoded))); - } - - } else { - if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { - xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); - byte[] raw = value.getData(); - byte[] compressed = MemUtil.compress(raw); - xml.text(String.valueOf(Base64Coder.encode(compressed))); - } else { - byte[] raw = value.getData(); - xml.text(String.valueOf(Base64Coder.encode(raw))); - } - - } + try (InputStream inputStream = value.getData()) { + if (inputStream == null) { + Log.e(TAG, "Can't write a null input stream."); + return; + } + + if (value.isProtected()) { + xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); + + try (InputStream cypherInputStream = + IOUtil.pipe(inputStream, + o -> new org.spongycastle.crypto.io.CipherOutputStream(o, randomStream))) { + writeInputStreamInBase64(cypherInputStream); + } + + } else { + if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); + + try (InputStream gZipInputStream = + IOUtil.pipe(inputStream, GZIPOutputStream::new, (int) value.length())) { + writeInputStreamInBase64(gZipInputStream); + } + + } else { + writeInputStreamInBase64(inputStream); + } + } + } } - + + private void writeInputStreamInBase64(InputStream inputStream) throws IOException { + try (InputStream base64InputStream = + IOUtil.pipe(inputStream, + o -> new Base64OutputStream(o, DEFAULT))) { + MemUtil.readBytes(base64InputStream, + buffer -> xml.text(Arrays.toString(buffer))); + } + } + //*/ + + //* + private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException { + + int valLength = (int) value.length(); + if (valLength > 0) { + byte[] buffer = new byte[valLength]; + if (valLength == value.getData().read(buffer, 0, valLength)) { + + if (value.isProtected()) { + xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue); + + byte[] encoded = new byte[valLength]; + randomStream.processBytes(buffer, 0, valLength, encoded, 0); + xml.text(String.valueOf(Base64Coder.encode(encoded))); + + } else { + if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) { + xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue); + + byte[] compressData = MemUtil.compress(buffer); + xml.text(String.valueOf(Base64Coder.encode(compressData))); + + } else { + xml.text(String.valueOf(Base64Coder.encode(buffer))); + } + } + } else { + Log.e(TAG, "Unable to read the stream of the protected binary"); + } + } + } + //*/ + private void writeObject(String name, String value, boolean filterXmlChars) throws IllegalArgumentException, IllegalStateException, IOException { assert(name != null && value != null); @@ -616,7 +664,7 @@ public class PwDbV4Output extends PwDbOutput { assert(binaries != null); for (Entry pair : binaries.entrySet()) { - writeObject(pair.getKey(), pair.getValue(), true); + writeObject(pair.getKey(), pair.getValue()); } } diff --git a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java index 463b7b604..2f1e373e2 100644 --- a/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java +++ b/app/src/main/java/com/kunzisoft/keepass/database/security/ProtectedBinary.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Brian Pellin, Jeremy Jamet / Kunzisoft. + * Copyright 2018 Jeremy Jamet / Kunzisoft. * * This file is part of KeePass DX. * @@ -21,52 +21,107 @@ package com.kunzisoft.keepass.database.security; import android.os.Parcel; import android.os.Parcelable; +import android.util.Log; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; public class ProtectedBinary implements Parcelable { - + + private static final String TAG = ProtectedBinary.class.getName(); public final static ProtectedBinary EMPTY = new ProtectedBinary(); private boolean protect; private byte[] data; - + private File dataFile; + private int size; + public boolean isProtected() { return protect; } - public int length() { - if (data == null) { - return 0; - } - return data.length; + public long length() { + if (data != null) + return data.length; + if (dataFile != null) + return size; + return 0; } - public ProtectedBinary() { - this(false, new byte[0]); + private ProtectedBinary() { + this.protect = false; + this.data = null; + this.dataFile = null; + this.size = 0; } public ProtectedBinary(boolean enableProtection, byte[] data) { this.protect = enableProtection; this.data = data; + this.dataFile = null; + if (data != null) + this.size = data.length; + else + this.size = 0; } - public ProtectedBinary(Parcel in) { + public ProtectedBinary(boolean enableProtection, File dataFile, int size) { + this.protect = enableProtection; + this.data = null; + this.dataFile = dataFile; + this.size = size; + } + + private ProtectedBinary(Parcel in) { protect = in.readByte() != 0; in.readByteArray(data); - } - - // TODO: replace the byte[] with something like ByteBuffer to make the return - // value immutable, so we don't have to worry about making deep copies - public byte[] getData() { - return data; - } - - public boolean equals(ProtectedBinary rhs) { - return (protect == rhs.protect) && Arrays.equals(data, rhs.data); + dataFile = new File(in.readString()); + size = in.readInt(); } - @Override + public InputStream getData() throws IOException { + if (data != null) + return new ByteArrayInputStream(data); + else if (dataFile != null) + return new FileInputStream(dataFile); + else + return null; + } + + public void clear() { + data = null; + if (dataFile != null && !dataFile.delete()) + Log.e(TAG, "Unable to delete temp file " + dataFile.getAbsolutePath()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ProtectedBinary that = (ProtectedBinary) o; + return protect == that.protect && + size == that.size && + Arrays.equals(data, that.data) && + dataFile != null && + dataFile.equals(that.dataFile); + } + + @Override + public int hashCode() { + + int result = 0; + result = 31 * result + (protect ? 1 : 0); + result = 31 * result + dataFile.hashCode(); + result = 31 * result + Integer.valueOf(size).hashCode(); + result = 31 * result + Arrays.hashCode(data); + return result; + } + + @Override public int describeContents() { return 0; } @@ -75,6 +130,8 @@ public class ProtectedBinary implements Parcelable { public void writeToParcel(Parcel dest, int flags) { dest.writeByte((byte) (protect ? 1 : 0)); dest.writeByteArray(data); + dest.writeString(dataFile.getAbsolutePath()); + dest.writeInt(size); } public static final Creator CREATOR = new Creator() { diff --git a/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java b/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java index 1db624afb..71c246737 100644 --- a/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java +++ b/app/src/main/java/com/kunzisoft/keepass/password/PasswordActivity.java @@ -252,7 +252,7 @@ public class PasswordActivity extends StylishActivity case LockingActivity.RESULT_EXIT_LOCK: case Activity.RESULT_CANCELED: setEmptyViews(); - App.getDB().clear(); + App.getDB().clear(getApplicationContext()); break; } } @@ -938,7 +938,7 @@ public class PasswordActivity extends StylishActivity private void loadDatabase(String password, Uri keyfile) { // Clear before we load Database database = App.getDB(); - database.clear(); + database.clear(getApplicationContext()); // Clear the shutdown flag App.clearShutdown(); diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java b/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java new file mode 100644 index 000000000..d72ecb2a4 --- /dev/null +++ b/app/src/main/java/com/kunzisoft/keepass/stream/ActionReadBytes.java @@ -0,0 +1,11 @@ +package com.kunzisoft.keepass.stream; + +import java.io.IOException; + +public interface ActionReadBytes { + /** + * Called after each buffer fill + * @param buffer filled + */ + void doAction(byte[] buffer) throws IOException; +} diff --git a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java index a213f7c76..1af163754 100644 --- a/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java +++ b/app/src/main/java/com/kunzisoft/keepass/stream/LEDataInputStream.java @@ -19,11 +19,9 @@ */ package com.kunzisoft.keepass.stream; -import com.kunzisoft.keepass.utils.Types; - import java.io.IOException; import java.io.InputStream; -import java.util.UUID; +import java.util.Arrays; /** Little endian version of the DataInputStream @@ -32,7 +30,7 @@ import java.util.UUID; */ public class LEDataInputStream extends InputStream { - public static final long INT_TO_LONG_MASK = 0xffffffffL; + private static final long INT_TO_LONG_MASK = 0xffffffffL; private InputStream baseStream; @@ -106,8 +104,9 @@ public class LEDataInputStream extends InputStream { } public byte[] readBytes(int length) throws IOException { + // TODO Exception max length < buffer size byte[] buf = new byte[length]; - + int count = 0; while ( count < length ) { int read = read(buf, count, length - count); @@ -126,6 +125,33 @@ public class LEDataInputStream extends InputStream { return buf; } + public void readBytes(int length, ActionReadBytes actionReadBytes) throws IOException { + int bufferSize = 256 * 3; // TODO Buffer size + byte[] buffer = new byte[bufferSize]; + + int offset = 0; + int read = 0; + while ( offset < length && read != -1) { + + // To reduce the buffer for the last bytes reads + if (length - offset < bufferSize) { + bufferSize = length - offset; + buffer = new byte[bufferSize]; + } + read = read(buffer, 0, bufferSize); + + // To get only the bytes read + byte[] optimizedBuffer; + if (read >= 0 && buffer.length > read) { + optimizedBuffer = Arrays.copyOf(buffer, read); + } else { + optimizedBuffer = buffer; + } + actionReadBytes.doAction(optimizedBuffer); + offset += read; + } + } + public static int readUShort(InputStream is) throws IOException { byte[] buf = new byte[2]; @@ -146,11 +172,11 @@ public class LEDataInputStream extends InputStream { * @return */ public static int readUShort( byte[] buf, int offset ) { - return (buf[offset + 0] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8); + return (buf[offset] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8); } public static long readLong( byte buf[], int offset ) { - return ((long)buf[offset + 0] & 0xFF) + (((long)buf[offset + 1] & 0xFF) << 8) + return ((long)buf[offset] & 0xFF) + (((long)buf[offset + 1] & 0xFF) << 8) + (((long)buf[offset + 2] & 0xFF) << 16) + (((long)buf[offset + 3] & 0xFF) << 24) + (((long)buf[offset + 4] & 0xFF) << 32) + (((long)buf[offset + 5] & 0xFF) << 40) + (((long)buf[offset + 6] & 0xFF) << 48) + (((long)buf[offset + 7] & 0xFF) << 56); @@ -180,14 +206,8 @@ public class LEDataInputStream extends InputStream { * @return */ public static int readInt( byte buf[], int offset ) { - return (buf[offset + 0] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8) + ((buf[offset + 2] & 0xFF) << 16) + return (buf[offset] & 0xFF) + ((buf[offset + 1] & 0xFF) << 8) + ((buf[offset + 2] & 0xFF) << 16) + ((buf[offset + 3] & 0xFF) << 24); } - public UUID readUUID() throws IOException { - byte[] buf = readBytes(16); - - return Types.bytestoUUID(buf); - } - } diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java index 92e516e86..42012767c 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/MemUtil.java @@ -21,38 +21,132 @@ package com.kunzisoft.keepass.utils; import android.os.Parcel; import android.os.Parcelable; +import android.support.annotation.NonNull; +import android.util.Log; + +import com.kunzisoft.keepass.stream.ActionReadBytes; + +import org.apache.commons.io.IOUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; public class MemUtil { + + private static final String TAG = MemUtil.class.getName(); + public static final int BUFFER_SIZE_BYTES = 3 * 128; + + public static void copyStream(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[BUFFER_SIZE_BYTES]; + int read; + try { + while ((read = in.read(buf)) != -1) { + out.write(buf, 0, read); + } + } catch (OutOfMemoryError error) { + throw new IOException(error); + } + } + + public static void readBytes(@NonNull InputStream inputStream, ActionReadBytes actionReadBytes) + throws IOException { + byte[] buffer = new byte[MemUtil.BUFFER_SIZE_BYTES]; + int read = 0; + while (read != -1) { + read = inputStream.read(buffer, 0, buffer.length); + if (read != -1) { + byte[] optimizedBuffer; + if (buffer.length == read) { + optimizedBuffer = buffer; + } else { + optimizedBuffer = Arrays.copyOf(buffer, read); + } + actionReadBytes.doAction(optimizedBuffer); + } + } + + /* + byte[] buffer = new byte[BUFFER_SIZE_BYTES]; + // To create the last buffer who is smaller + long numberOfFullBuffer = length / buffer.length; + long sizeOfFullBuffers = numberOfFullBuffer * buffer.length; + int read = 0; + //if (protectedBinary.length() > 0) { + while (read < length) { + // Create the last smaller buffer + if (read >= sizeOfFullBuffers) + buffer = new byte[(int) (length % buffer.length)]; + read += inputStream.read(buffer, 0, buffer.length); + actionReadBytes.doAction(buffer); + } + //*/ + } + public static byte[] decompress(byte[] input) throws IOException { ByteArrayInputStream bais = new ByteArrayInputStream(input); GZIPInputStream gzis = new GZIPInputStream(bais); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyStream(gzis, baos); - - return baos.toByteArray(); - } - - public static byte[] compress(byte[] input) throws IOException { - ByteArrayInputStream bais = new ByteArrayInputStream(input); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - GZIPOutputStream gzos = new GZIPOutputStream(baos); - Util.copyStream(bais, gzos); - gzos.close(); + copyStream(gzis, baos); return baos.toByteArray(); } - // For writing to a Parcel + public static byte[] compress(byte[] input) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(input); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream gzos = new GZIPOutputStream(baos); + copyStream(bais, gzos); + gzos.close(); + + return baos.toByteArray(); + } + + /** + * Compresses the input data using GZip and outputs the compressed data. + * + * @param input + * An {@link InputStream} containing the input raw data. + * + * @return An {@link InputStream} to the compressed data. + */ + public static InputStream compress(final InputStream input) { + final PipedInputStream compressedDataStream = new PipedInputStream(3 * 128); + Log.d(TAG, "About to compress input data using gzip asynchronously..."); + PipedOutputStream compressionOutput; + GZIPOutputStream gzipCompressedDataStream = null; + try { + compressionOutput = new PipedOutputStream(compressedDataStream); + gzipCompressedDataStream = new GZIPOutputStream(compressionOutput); + IOUtils.copy(input, gzipCompressedDataStream); + Log.e(TAG, "Successfully compressed input data using gzip."); + } catch (IOException e) { + Log.e(TAG, "Failed to compress input data.", e); + } finally { + if (gzipCompressedDataStream != null) { + try { + gzipCompressedDataStream.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close gzip output stream.", e); + } + } + } + return compressedDataStream; + } + + + // For writing to a Parcel public static void writeParcelableMap( Parcel parcel, int flags, Map map) { parcel.writeInt(map.size()); diff --git a/app/src/main/java/com/kunzisoft/keepass/utils/Util.java b/app/src/main/java/com/kunzisoft/keepass/utils/Util.java index c6b90e84d..ae8c899f0 100644 --- a/app/src/main/java/com/kunzisoft/keepass/utils/Util.java +++ b/app/src/main/java/com/kunzisoft/keepass/utils/Util.java @@ -27,16 +27,11 @@ import android.content.pm.ActivityInfo; import android.content.res.Configuration; import android.graphics.Typeface; import android.net.Uri; -import android.util.TypedValue; import android.widget.EditText; import android.widget.TextView; import com.kunzisoft.keepass.R; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - public class Util { public static void gotoUrl(Context context, String url) throws ActivityNotFoundException { @@ -49,20 +44,8 @@ public class Util { public static void gotoUrl(Context context, int resId) throws ActivityNotFoundException { gotoUrl(context, context.getString(resId)); } - - public static void copyStream(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[1024]; - int read; - try { - while ((read = in.read(buf)) != -1) { - out.write(buf, 0, read); - } - } catch (OutOfMemoryError error) { - throw new IOException(error); - } - } - /** + /** * Replace font by monospace, must be called after seText() */ public static void applyFontVisibilityTo(final Context context, final TextView textView) {