Optimize memory pool and revert algo

This commit is contained in:
J-Jamet committed 2018-09-18 10:11:44 +02:00
1 parent b96436c906
commit dc23c6a445
4 files changed
+176 -71

No files matched your search

@@ -23,7 +23,6 @@ import com.kunzisoft.keepass.R;
import com.kunzisoft.keepass.crypto.CipherFactory;
import com.kunzisoft.keepass.crypto.PwStreamCipherFactory;
import com.kunzisoft.keepass.crypto.engine.CipherEngine;
import com.kunzisoft.keepass.database.BinaryPool;
import com.kunzisoft.keepass.database.ITimeLogger;
import com.kunzisoft.keepass.database.PwCompressionAlgorithm;
import com.kunzisoft.keepass.database.PwDatabase;
@@ -211,7 +210,10 @@ public class ImporterV4 extends Importer {
while(true) {
if (!ReadInnerHeader(lis, header)) break;
}
}
private String getUnusedCacheFileName() {
return String.valueOf(db.getBinPool().findUnusedKey());
}
private boolean ReadInnerHeader(LEDataInputStream lis, PwDbHeaderV4 header) throws IOException {
@@ -241,15 +243,14 @@ public class ImporterV4 extends Importer {
byte flag = lis.readBytes(1)[0];
boolean protectedFlag = (flag & PwDbHeaderV4.KdbxBinaryFlags.Protected) !=
PwDbHeaderV4.KdbxBinaryFlags.None;
int byteLength = size - 1;
// Read in a file
BinaryPool binaryPool = db.getBinPool();
int binaryKey = binaryPool.findUnusedKey();
File file = new File(streamDir, String.valueOf(binaryKey));
File file = new File(streamDir, getUnusedCacheFileName());
try (FileOutputStream outputStream = new FileOutputStream(file)) {
lis.readBytes(size - 1, outputStream::write);
lis.readBytes(byteLength, outputStream::write);
}
ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, size -1);
binaryPool.add(protectedBinary);
ProtectedBinary protectedBinary = new ProtectedBinary(protectedFlag, file, byteLength);
db.getBinPool().add(protectedBinary);
break;
default:
@@ -288,9 +289,9 @@ public class ImporterV4 extends Importer {
}
private static final long DEFAULT_HISTORY_DAYS = 365;
private boolean readNextNode = true;
private Stack<PwGroupV4> ctxGroups = new Stack<PwGroupV4>();
private Stack<PwGroupV4> ctxGroups = new Stack<>();
private PwGroupV4 ctxGroup = null;
private PwEntryV4 ctxEntry = null;
private String ctxStringName = null;
@@ -336,7 +337,7 @@ public class ImporterV4 extends Importer {
ctxGroups.clear();
KdbContext ctx = KdbContext.Null;
readNextNode = true;
while (true) {
@@ -1027,7 +1028,19 @@ public class ImporterV4 extends Importer {
return new ProtectedString(false, ReadString(xpp));
}
private ProtectedBinary createProtectedBinaryFromData(boolean protection, byte[] data) throws IOException {
if (data.length > MemUtil.BUFFER_SIZE_BYTES) {
File file = new File(streamDir, getUnusedCacheFileName());
try (FileOutputStream outputStream = new FileOutputStream(file)) {
outputStream.write(data);
}
return new ProtectedBinary(protection, file, data.length);
} else {
return new ProtectedBinary(protection, data);
}
}
private ProtectedBinary ReadProtectedBinary(XmlPullParser xpp) throws XmlPullParserException, IOException {
String ref = xpp.getAttributeValue(null, PwDatabaseV4XML.AttrRef);
if (ref != null) {
@@ -1045,8 +1058,9 @@ public class ImporterV4 extends Importer {
byte[] buf = ProcessNode(xpp);
if ( buf != null )
return new ProtectedBinary(true, buf);
if ( buf != null ) {
createProtectedBinaryFromData(true, buf);
}
String base64 = ReadString(xpp);
if ( base64.length() == 0 )
@@ -1058,7 +1072,7 @@ public class ImporterV4 extends Importer {
data = MemUtil.decompress(data);
}
return new ProtectedBinary(false, data);
return createProtectedBinaryFromData(false, data);
}
private String ReadString(XmlPullParser xpp) throws IOException, XmlPullParserException {
@@ -23,9 +23,9 @@ import com.kunzisoft.keepass.database.PwDatabaseV4;
import com.kunzisoft.keepass.database.PwDbHeaderV4;
import com.kunzisoft.keepass.database.security.ProtectedBinary;
import com.kunzisoft.keepass.stream.LEDataOutputStream;
import com.kunzisoft.keepass.utils.MemUtil;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public class PwDbInnerHeaderOutputV4 {
@@ -60,26 +60,8 @@ public class PwDbInnerHeaderOutputV4 {
los.writeInt((int) protectedBinary.length() + 1); // TODO verify
los.write(flag);
/*
MemUtil.readBytes(protectedBinary.getData(),
(int) protectedBinary.length(),
buffer -> los.write(buffer));
*/
byte[] buffer = new byte[3 * 256]; // TODO buffer generalize
InputStream fileInputStream = protectedBinary.getData(); // TODO Nullable
// To create the last buffer who is smaller
long numberOfFullBuffer = protectedBinary.length() / buffer.length;
long sizeOfFullBuffers = numberOfFullBuffer * buffer.length;
int read = 0;
//if (protectedBinary.length() > 0) {
while (read < protectedBinary.length()) {
// Create the last smaller buffer
if (read >= sizeOfFullBuffers)
buffer = new byte[(int) (protectedBinary.length() % buffer.length)];
read += fileInputStream.read(buffer, 0, buffer.length);
los.write(buffer);
}
}
los.write(PwDbHeaderV4.PwDbInnerHeaderV4Fields.EndOfHeader);
@@ -394,9 +394,8 @@ public class PwDbV4Output extends PwDbOutput<PwDbHeaderV4> {
if (!isHistory) {
writeList(PwDatabaseV4XML.ElemHistory, entry.getHistory(), true);
} else {
assert(entry.sizeOfHistory() == 0);
}
// else entry.sizeOfHistory() == 0
xml.endTag(null, PwDatabaseV4XML.ElemEntry);
}
@@ -424,54 +423,85 @@ public class PwDbV4Output extends PwDbOutput<PwDbHeaderV4> {
xml.endTag(null, PwDatabaseV4XML.ElemBinary);
}
private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException {
int valLength = (int) value.length(); // TODO verify
if (valLength > 0) {
byte[] buffer = new byte[valLength];
value.getData().read(buffer, 0, valLength); // TODO Nullable
/*
TODO Make with pipe
private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException {
try (InputStream inputStream = value.getData()) {
if (inputStream == null) {
Log.e(TAG, "Can't write a null input stream.");
return;
}
if (value.isProtected()) {
xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue);
byte[] encoded = new byte[valLength];
randomStream.processBytes(buffer, 0, valLength, encoded, 0);
xml.text(String.valueOf(Base64Coder.encode(encoded)));
try (InputStream cypherInputStream =
IOUtil.pipe(inputStream,
o -> new org.spongycastle.crypto.io.CipherOutputStream(o, randomStream))) {
writeInputStreamInBase64(cypherInputStream);
}
} else {
if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) {
xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue);
byte[] compressData = MemUtil.compress(buffer);
//InputStream gzipInputStream = IOUtil.pipe(fileInputStream, GZIPOutputStream::new, valLength); // TODO buffer generalize
//gzipInputStream.read(buffer, 0, valLength);
xml.text(String.valueOf(Base64Coder.encode(compressData))); // TODO Compress all
/*
// TODO be a problem not the same length
MemUtil.readBytes(gzipInputStream, value.length(), new ActionReadBytes() {
@Override
public void doAction(byte[] buffer) throws IOException {
xml.text(String.valueOf(Base64Coder.encode(buffer)));
}
});
*/
}
else {
xml.text(String.valueOf(Base64Coder.encode(buffer)));
/*
// TODO Debug
MemUtil.readBytes(value.getData(),
(int) value.length(),
buffer -> xml.text(String.valueOf(Base64Coder.encode(buffer))));
*/
try (InputStream gZipInputStream =
IOUtil.pipe(inputStream, GZIPOutputStream::new, (int) value.length())) {
writeInputStreamInBase64(gZipInputStream);
}
} else {
writeInputStreamInBase64(inputStream);
}
}
// TODO remove buffer
}
}
private void writeInputStreamInBase64(InputStream inputStream) throws IOException {
try (InputStream base64InputStream =
IOUtil.pipe(inputStream,
o -> new Base64OutputStream(o, DEFAULT))) {
MemUtil.readBytes(base64InputStream,
buffer -> xml.text(Arrays.toString(buffer)));
}
}
//*/
//*
private void subWriteValue(ProtectedBinary value) throws IllegalArgumentException, IllegalStateException, IOException {
int valLength = (int) value.length();
if (valLength > 0) {
byte[] buffer = new byte[valLength];
if (valLength == value.getData().read(buffer, 0, valLength)) {
if (value.isProtected()) {
xml.attribute(null, PwDatabaseV4XML.AttrProtected, PwDatabaseV4XML.ValTrue);
byte[] encoded = new byte[valLength];
randomStream.processBytes(buffer, 0, valLength, encoded, 0);
xml.text(String.valueOf(Base64Coder.encode(encoded)));
} else {
if (mPM.getCompressionAlgorithm() == PwCompressionAlgorithm.Gzip) {
xml.attribute(null, PwDatabaseV4XML.AttrCompressed, PwDatabaseV4XML.ValTrue);
byte[] compressData = MemUtil.compress(buffer);
xml.text(String.valueOf(Base64Coder.encode(compressData)));
} else {
xml.text(String.valueOf(Base64Coder.encode(buffer)));
}
}
} else {
Log.e(TAG, "Unable to read the stream of the protected binary");
}
}
}
//*/
private void writeObject(String name, String value, boolean filterXmlChars) throws IllegalArgumentException, IllegalStateException, IOException {
assert(name != null && value != null);
@@ -21,12 +21,21 @@ package com.kunzisoft.keepass.utils;
import android.os.Parcel;
import android.os.Parcelable;
import android.support.annotation.NonNull;
import android.util.Log;
import com.kunzisoft.keepass.stream.ActionReadBytes;
import org.apache.commons.io.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.zip.GZIPInputStream;
@@ -34,8 +43,11 @@ import java.util.zip.GZIPOutputStream;
public class MemUtil {
private static final String TAG = MemUtil.class.getName();
public static final int BUFFER_SIZE_BYTES = 3 * 128;
public static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] buf = new byte[1024];
byte[] buf = new byte[BUFFER_SIZE_BYTES];
int read;
try {
while ((read = in.read(buf)) != -1) {
@@ -46,6 +58,40 @@ public class MemUtil {
}
}
public static void readBytes(@NonNull InputStream inputStream, ActionReadBytes actionReadBytes)
throws IOException {
byte[] buffer = new byte[MemUtil.BUFFER_SIZE_BYTES];
int read = 0;
while (read != -1) {
read = inputStream.read(buffer, 0, buffer.length);
if (read != -1) {
byte[] optimizedBuffer;
if (buffer.length == read) {
optimizedBuffer = buffer;
} else {
optimizedBuffer = Arrays.copyOf(buffer, read);
}
actionReadBytes.doAction(optimizedBuffer);
}
}
/*
byte[] buffer = new byte[BUFFER_SIZE_BYTES];
// To create the last buffer who is smaller
long numberOfFullBuffer = length / buffer.length;
long sizeOfFullBuffers = numberOfFullBuffer * buffer.length;
int read = 0;
//if (protectedBinary.length() > 0) {
while (read < length) {
// Create the last smaller buffer
if (read >= sizeOfFullBuffers)
buffer = new byte[(int) (length % buffer.length)];
read += inputStream.read(buffer, 0, buffer.length);
actionReadBytes.doAction(buffer);
}
//*/
}
public static byte[] decompress(byte[] input) throws IOException {
ByteArrayInputStream bais = new ByteArrayInputStream(input);
GZIPInputStream gzis = new GZIPInputStream(bais);
@@ -67,7 +113,40 @@ public class MemUtil {
return baos.toByteArray();
}
// For writing to a Parcel
/**
* Compresses the input data using GZip and outputs the compressed data.
*
* @param input
* An {@link InputStream} containing the input raw data.
*
* @return An {@link InputStream} to the compressed data.
*/
public static InputStream compress(final InputStream input) {
final PipedInputStream compressedDataStream = new PipedInputStream(3 * 128);
Log.d(TAG, "About to compress input data using gzip asynchronously...");
PipedOutputStream compressionOutput;
GZIPOutputStream gzipCompressedDataStream = null;
try {
compressionOutput = new PipedOutputStream(compressedDataStream);
gzipCompressedDataStream = new GZIPOutputStream(compressionOutput);
IOUtils.copy(input, gzipCompressedDataStream);
Log.e(TAG, "Successfully compressed input data using gzip.");
} catch (IOException e) {
Log.e(TAG, "Failed to compress input data.", e);
} finally {
if (gzipCompressedDataStream != null) {
try {
gzipCompressedDataStream.close();
} catch (IOException e) {
Log.e(TAG, "Failed to close gzip output stream.", e);
}
}
}
return compressedDataStream;
}
// For writing to a Parcel
public static <K extends Parcelable,V extends Parcelable> void writeParcelableMap(
Parcel parcel, int flags, Map<K, V > map) {
parcel.writeInt(map.size());